]>
Commit | Line | Data |
---|---|---|
a9d6c7fb | 1 | /* Lower _BitInt(N) operations to scalar operations. |
a945c346 | 2 | Copyright (C) 2023-2024 Free Software Foundation, Inc. |
a9d6c7fb JJ |
3 | Contributed by Jakub Jelinek <jakub@redhat.com>. |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it | |
8 | under the terms of the GNU General Public License as published by the | |
9 | Free Software Foundation; either version 3, or (at your option) any | |
10 | later version. | |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT | |
13 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
18 | along with GCC; see the file COPYING3. If not see | |
19 | <http://www.gnu.org/licenses/>. */ | |
20 | ||
21 | #include "config.h" | |
22 | #include "system.h" | |
23 | #include "coretypes.h" | |
24 | #include "backend.h" | |
25 | #include "rtl.h" | |
26 | #include "tree.h" | |
27 | #include "gimple.h" | |
28 | #include "cfghooks.h" | |
29 | #include "tree-pass.h" | |
30 | #include "ssa.h" | |
31 | #include "fold-const.h" | |
32 | #include "gimplify.h" | |
33 | #include "gimple-iterator.h" | |
34 | #include "tree-cfg.h" | |
35 | #include "tree-dfa.h" | |
36 | #include "cfgloop.h" | |
37 | #include "cfganal.h" | |
38 | #include "target.h" | |
39 | #include "tree-ssa-live.h" | |
40 | #include "tree-ssa-coalesce.h" | |
41 | #include "domwalk.h" | |
42 | #include "memmodel.h" | |
43 | #include "optabs.h" | |
44 | #include "varasm.h" | |
45 | #include "gimple-range.h" | |
46 | #include "value-range.h" | |
47 | #include "langhooks.h" | |
48 | #include "gimplify-me.h" | |
49 | #include "diagnostic-core.h" | |
50 | #include "tree-eh.h" | |
51 | #include "tree-pretty-print.h" | |
52 | #include "alloc-pool.h" | |
53 | #include "tree-into-ssa.h" | |
54 | #include "tree-cfgcleanup.h" | |
55 | #include "tree-switch-conversion.h" | |
56 | #include "ubsan.h" | |
9d2bc5de | 57 | #include "stor-layout.h" |
a9d6c7fb JJ |
58 | #include "gimple-lower-bitint.h" |
59 | ||
60 | /* Split BITINT_TYPE precisions in 4 categories. Small _BitInt, where | |
61 | target hook says it is a single limb, middle _BitInt which per ABI | |
62 | does not, but there is some INTEGER_TYPE in which arithmetics can be | |
63 | performed (operations on such _BitInt are lowered to casts to that | |
64 | arithmetic type and cast back; e.g. on x86_64 limb is DImode, but | |
65 | target supports TImode, so _BitInt(65) to _BitInt(128) are middle | |
66 | ones), large _BitInt which should by straight line code and | |
67 | finally huge _BitInt which should be handled by loops over the limbs. */ | |
68 | ||
69 | enum bitint_prec_kind { | |
70 | bitint_prec_small, | |
71 | bitint_prec_middle, | |
72 | bitint_prec_large, | |
73 | bitint_prec_huge | |
74 | }; | |
75 | ||
76 | /* Caches to speed up bitint_precision_kind. */ | |
77 | ||
78 | static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec; | |
79 | static int limb_prec; | |
80 | ||
81 | /* Categorize _BitInt(PREC) as small, middle, large or huge. */ | |
82 | ||
83 | static bitint_prec_kind | |
84 | bitint_precision_kind (int prec) | |
85 | { | |
86 | if (prec <= small_max_prec) | |
87 | return bitint_prec_small; | |
88 | if (huge_min_prec && prec >= huge_min_prec) | |
89 | return bitint_prec_huge; | |
90 | if (large_min_prec && prec >= large_min_prec) | |
91 | return bitint_prec_large; | |
92 | if (mid_min_prec && prec >= mid_min_prec) | |
93 | return bitint_prec_middle; | |
94 | ||
95 | struct bitint_info info; | |
18c90eaa JJ |
96 | bool ok = targetm.c.bitint_type_info (prec, &info); |
97 | gcc_assert (ok); | |
a9d6c7fb JJ |
98 | scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode); |
99 | if (prec <= GET_MODE_PRECISION (limb_mode)) | |
100 | { | |
101 | small_max_prec = prec; | |
102 | return bitint_prec_small; | |
103 | } | |
a9d6c7fb | 104 | if (!large_min_prec |
47ecac52 JJ |
105 | && GET_MODE_PRECISION (limb_mode) < MAX_FIXED_MODE_SIZE) |
106 | large_min_prec = MAX_FIXED_MODE_SIZE + 1; | |
a9d6c7fb JJ |
107 | if (!limb_prec) |
108 | limb_prec = GET_MODE_PRECISION (limb_mode); | |
109 | if (!huge_min_prec) | |
110 | { | |
47ecac52 | 111 | if (4 * limb_prec >= MAX_FIXED_MODE_SIZE) |
a9d6c7fb JJ |
112 | huge_min_prec = 4 * limb_prec; |
113 | else | |
47ecac52 | 114 | huge_min_prec = MAX_FIXED_MODE_SIZE + 1; |
a9d6c7fb | 115 | } |
47ecac52 | 116 | if (prec <= MAX_FIXED_MODE_SIZE) |
a9d6c7fb JJ |
117 | { |
118 | if (!mid_min_prec || prec < mid_min_prec) | |
119 | mid_min_prec = prec; | |
120 | return bitint_prec_middle; | |
121 | } | |
122 | if (large_min_prec && prec <= large_min_prec) | |
123 | return bitint_prec_large; | |
124 | return bitint_prec_huge; | |
125 | } | |
126 | ||
127 | /* Same for a TYPE. */ | |
128 | ||
129 | static bitint_prec_kind | |
130 | bitint_precision_kind (tree type) | |
131 | { | |
132 | return bitint_precision_kind (TYPE_PRECISION (type)); | |
133 | } | |
134 | ||
135 | /* Return minimum precision needed to describe INTEGER_CST | |
136 | CST. All bits above that precision up to precision of | |
137 | TREE_TYPE (CST) are cleared if EXT is set to 0, or set | |
138 | if EXT is set to -1. */ | |
139 | ||
140 | static unsigned | |
141 | bitint_min_cst_precision (tree cst, int &ext) | |
142 | { | |
143 | ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0; | |
144 | wide_int w = wi::to_wide (cst); | |
145 | unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst))); | |
146 | /* For signed values, we don't need to count the sign bit, | |
147 | we'll use constant 0 or -1 for the upper bits. */ | |
148 | if (!TYPE_UNSIGNED (TREE_TYPE (cst))) | |
149 | --min_prec; | |
150 | else | |
151 | { | |
152 | /* For unsigned values, also try signed min_precision | |
153 | in case the constant has lots of most significant bits set. */ | |
154 | unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1; | |
155 | if (min_prec2 < min_prec) | |
156 | { | |
157 | ext = -1; | |
158 | return min_prec2; | |
159 | } | |
160 | } | |
161 | return min_prec; | |
162 | } | |
163 | ||
164 | namespace { | |
165 | ||
166 | /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE | |
167 | cached in TYPE and return it. */ | |
168 | ||
169 | tree | |
170 | maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type) | |
171 | { | |
172 | if (op == NULL_TREE | |
173 | || TREE_CODE (TREE_TYPE (op)) != BITINT_TYPE | |
174 | || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle) | |
175 | return op; | |
176 | ||
177 | int prec = TYPE_PRECISION (TREE_TYPE (op)); | |
178 | int uns = TYPE_UNSIGNED (TREE_TYPE (op)); | |
179 | if (type == NULL_TREE | |
180 | || TYPE_PRECISION (type) != prec | |
181 | || TYPE_UNSIGNED (type) != uns) | |
182 | type = build_nonstandard_integer_type (prec, uns); | |
183 | ||
184 | if (TREE_CODE (op) != SSA_NAME) | |
185 | { | |
186 | tree nop = fold_convert (type, op); | |
187 | if (is_gimple_val (nop)) | |
188 | return nop; | |
189 | } | |
190 | ||
191 | tree nop = make_ssa_name (type); | |
192 | gimple *g = gimple_build_assign (nop, NOP_EXPR, op); | |
193 | gsi_insert_before (gsi, g, GSI_SAME_STMT); | |
194 | return nop; | |
195 | } | |
196 | ||
197 | /* Return true if STMT can be handled in a loop from least to most | |
198 | significant limb together with its dependencies. */ | |
199 | ||
200 | bool | |
201 | mergeable_op (gimple *stmt) | |
202 | { | |
203 | if (!is_gimple_assign (stmt)) | |
204 | return false; | |
205 | switch (gimple_assign_rhs_code (stmt)) | |
206 | { | |
207 | case PLUS_EXPR: | |
208 | case MINUS_EXPR: | |
209 | case NEGATE_EXPR: | |
210 | case BIT_AND_EXPR: | |
211 | case BIT_IOR_EXPR: | |
212 | case BIT_XOR_EXPR: | |
213 | case BIT_NOT_EXPR: | |
214 | case SSA_NAME: | |
215 | case INTEGER_CST: | |
9d2bc5de | 216 | case BIT_FIELD_REF: |
a9d6c7fb JJ |
217 | return true; |
218 | case LSHIFT_EXPR: | |
219 | { | |
220 | tree cnt = gimple_assign_rhs2 (stmt); | |
221 | if (tree_fits_uhwi_p (cnt) | |
222 | && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec) | |
223 | return true; | |
224 | } | |
225 | break; | |
226 | CASE_CONVERT: | |
227 | case VIEW_CONVERT_EXPR: | |
228 | { | |
229 | tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt)); | |
230 | tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt)); | |
231 | if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME | |
232 | && TREE_CODE (lhs_type) == BITINT_TYPE | |
233 | && TREE_CODE (rhs_type) == BITINT_TYPE | |
234 | && bitint_precision_kind (lhs_type) >= bitint_prec_large | |
235 | && bitint_precision_kind (rhs_type) >= bitint_prec_large | |
4ff5e7ce JJ |
236 | && (CEIL (TYPE_PRECISION (lhs_type), limb_prec) |
237 | == CEIL (TYPE_PRECISION (rhs_type), limb_prec))) | |
a9d6c7fb JJ |
238 | { |
239 | if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)) | |
240 | return true; | |
241 | if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0) | |
242 | return true; | |
243 | if (bitint_precision_kind (lhs_type) == bitint_prec_large) | |
244 | return true; | |
245 | } | |
246 | break; | |
247 | } | |
248 | default: | |
249 | break; | |
250 | } | |
251 | return false; | |
252 | } | |
253 | ||
254 | /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with | |
255 | _Complex large/huge _BitInt lhs which has at most two immediate uses, | |
256 | at most one use in REALPART_EXPR stmt in the same bb and exactly one | |
257 | IMAGPART_EXPR use in the same bb with a single use which casts it to | |
258 | non-BITINT_TYPE integral type. If there is a REALPART_EXPR use, | |
259 | return 2. Such cases (most common uses of those builtins) can be | |
260 | optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs | |
261 | of REALPART_EXPR as not needed to be backed up by a stack variable. | |
262 | For .UBSAN_CHECK_{ADD,SUB,MUL} return 3. */ | |
263 | ||
264 | int | |
265 | optimizable_arith_overflow (gimple *stmt) | |
266 | { | |
267 | bool is_ubsan = false; | |
268 | if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt)) | |
269 | return false; | |
270 | switch (gimple_call_internal_fn (stmt)) | |
271 | { | |
272 | case IFN_ADD_OVERFLOW: | |
273 | case IFN_SUB_OVERFLOW: | |
274 | case IFN_MUL_OVERFLOW: | |
275 | break; | |
276 | case IFN_UBSAN_CHECK_ADD: | |
277 | case IFN_UBSAN_CHECK_SUB: | |
278 | case IFN_UBSAN_CHECK_MUL: | |
279 | is_ubsan = true; | |
280 | break; | |
281 | default: | |
282 | return 0; | |
283 | } | |
284 | tree lhs = gimple_call_lhs (stmt); | |
285 | if (!lhs) | |
286 | return 0; | |
287 | if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs)) | |
288 | return 0; | |
289 | tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs)); | |
290 | if (TREE_CODE (type) != BITINT_TYPE | |
291 | || bitint_precision_kind (type) < bitint_prec_large) | |
292 | return 0; | |
293 | ||
294 | if (is_ubsan) | |
295 | { | |
296 | use_operand_p use_p; | |
297 | gimple *use_stmt; | |
298 | if (!single_imm_use (lhs, &use_p, &use_stmt) | |
299 | || gimple_bb (use_stmt) != gimple_bb (stmt) | |
300 | || !gimple_store_p (use_stmt) | |
301 | || !is_gimple_assign (use_stmt) | |
302 | || gimple_has_volatile_ops (use_stmt) | |
303 | || stmt_ends_bb_p (use_stmt)) | |
304 | return 0; | |
305 | return 3; | |
306 | } | |
307 | ||
308 | imm_use_iterator ui; | |
309 | use_operand_p use_p; | |
310 | int seen = 0; | |
7590d975 | 311 | gimple *realpart = NULL, *cast = NULL; |
a9d6c7fb JJ |
312 | FOR_EACH_IMM_USE_FAST (use_p, ui, lhs) |
313 | { | |
314 | gimple *g = USE_STMT (use_p); | |
315 | if (is_gimple_debug (g)) | |
316 | continue; | |
317 | if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt)) | |
318 | return 0; | |
319 | if (gimple_assign_rhs_code (g) == REALPART_EXPR) | |
320 | { | |
321 | if ((seen & 1) != 0) | |
322 | return 0; | |
323 | seen |= 1; | |
7590d975 | 324 | realpart = g; |
a9d6c7fb JJ |
325 | } |
326 | else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR) | |
327 | { | |
328 | if ((seen & 2) != 0) | |
329 | return 0; | |
330 | seen |= 2; | |
331 | ||
332 | use_operand_p use2_p; | |
333 | gimple *use_stmt; | |
334 | tree lhs2 = gimple_assign_lhs (g); | |
335 | if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2)) | |
336 | return 0; | |
337 | if (!single_imm_use (lhs2, &use2_p, &use_stmt) | |
338 | || gimple_bb (use_stmt) != gimple_bb (stmt) | |
339 | || !gimple_assign_cast_p (use_stmt)) | |
340 | return 0; | |
341 | ||
342 | lhs2 = gimple_assign_lhs (use_stmt); | |
343 | if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2)) | |
344 | || TREE_CODE (TREE_TYPE (lhs2)) == BITINT_TYPE) | |
345 | return 0; | |
7590d975 | 346 | cast = use_stmt; |
a9d6c7fb JJ |
347 | } |
348 | else | |
349 | return 0; | |
350 | } | |
351 | if ((seen & 2) == 0) | |
352 | return 0; | |
7590d975 JJ |
353 | if (seen == 3) |
354 | { | |
355 | /* Punt if the cast stmt appears before realpart stmt, because | |
356 | if both appear, the lowering wants to emit all the code | |
357 | at the location of realpart stmt. */ | |
358 | gimple_stmt_iterator gsi = gsi_for_stmt (realpart); | |
359 | unsigned int cnt = 0; | |
360 | do | |
361 | { | |
362 | gsi_prev_nondebug (&gsi); | |
363 | if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast) | |
364 | return 0; | |
365 | if (gsi_stmt (gsi) == stmt) | |
366 | return 2; | |
367 | /* If realpart is too far from stmt, punt as well. | |
368 | Usually it will appear right after it. */ | |
369 | if (++cnt == 32) | |
370 | return 0; | |
371 | } | |
372 | while (1); | |
373 | } | |
374 | return 1; | |
a9d6c7fb JJ |
375 | } |
376 | ||
377 | /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment) | |
378 | comparing large/huge _BitInt types, return the comparison code and if | |
379 | non-NULL fill in the comparison operands to *POP1 and *POP2. */ | |
380 | ||
381 | tree_code | |
382 | comparison_op (gimple *stmt, tree *pop1, tree *pop2) | |
383 | { | |
384 | tree op1 = NULL_TREE, op2 = NULL_TREE; | |
385 | tree_code code = ERROR_MARK; | |
386 | if (gimple_code (stmt) == GIMPLE_COND) | |
387 | { | |
388 | code = gimple_cond_code (stmt); | |
389 | op1 = gimple_cond_lhs (stmt); | |
390 | op2 = gimple_cond_rhs (stmt); | |
391 | } | |
392 | else if (is_gimple_assign (stmt)) | |
393 | { | |
394 | code = gimple_assign_rhs_code (stmt); | |
395 | op1 = gimple_assign_rhs1 (stmt); | |
396 | if (TREE_CODE_CLASS (code) == tcc_comparison | |
397 | || TREE_CODE_CLASS (code) == tcc_binary) | |
398 | op2 = gimple_assign_rhs2 (stmt); | |
399 | } | |
400 | if (TREE_CODE_CLASS (code) != tcc_comparison) | |
401 | return ERROR_MARK; | |
402 | tree type = TREE_TYPE (op1); | |
403 | if (TREE_CODE (type) != BITINT_TYPE | |
404 | || bitint_precision_kind (type) < bitint_prec_large) | |
405 | return ERROR_MARK; | |
406 | if (pop1) | |
407 | { | |
408 | *pop1 = op1; | |
409 | *pop2 = op2; | |
410 | } | |
411 | return code; | |
412 | } | |
413 | ||
414 | /* Class used during large/huge _BitInt lowering containing all the | |
415 | state for the methods. */ | |
416 | ||
417 | struct bitint_large_huge | |
418 | { | |
419 | bitint_large_huge () | |
420 | : m_names (NULL), m_loads (NULL), m_preserved (NULL), | |
421 | m_single_use_names (NULL), m_map (NULL), m_vars (NULL), | |
90b98723 JJ |
422 | m_limb_type (NULL_TREE), m_data (vNULL), |
423 | m_returns_twice_calls (vNULL) {} | |
a9d6c7fb JJ |
424 | |
425 | ~bitint_large_huge (); | |
426 | ||
427 | void insert_before (gimple *); | |
428 | tree limb_access_type (tree, tree); | |
429 | tree limb_access (tree, tree, tree, bool); | |
3e8323e8 JJ |
430 | tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT, |
431 | unsigned HOST_WIDE_INT); | |
a9d6c7fb JJ |
432 | void if_then (gimple *, profile_probability, edge &, edge &); |
433 | void if_then_else (gimple *, profile_probability, edge &, edge &); | |
434 | void if_then_if_then_else (gimple *g, gimple *, | |
435 | profile_probability, profile_probability, | |
436 | edge &, edge &, edge &); | |
437 | tree handle_operand (tree, tree); | |
07efd566 | 438 | tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE); |
a9d6c7fb JJ |
439 | tree add_cast (tree, tree); |
440 | tree handle_plus_minus (tree_code, tree, tree, tree); | |
441 | tree handle_lshift (tree, tree, tree); | |
442 | tree handle_cast (tree, tree, tree); | |
9d2bc5de | 443 | tree handle_bit_field_ref (tree, tree); |
a9d6c7fb JJ |
444 | tree handle_load (gimple *, tree); |
445 | tree handle_stmt (gimple *, tree); | |
446 | tree handle_operand_addr (tree, gimple *, int *, int *); | |
447 | tree create_loop (tree, tree *); | |
448 | tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree); | |
449 | tree lower_comparison_stmt (gimple *, tree_code &, tree, tree); | |
450 | void lower_shift_stmt (tree, gimple *); | |
451 | void lower_muldiv_stmt (tree, gimple *); | |
452 | void lower_float_conv_stmt (tree, gimple *); | |
453 | tree arith_overflow_extract_bits (unsigned int, unsigned int, tree, | |
454 | unsigned int, bool); | |
455 | void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *, | |
456 | tree_code); | |
457 | void lower_addsub_overflow (tree, gimple *); | |
458 | void lower_mul_overflow (tree, gimple *); | |
459 | void lower_cplxpart_stmt (tree, gimple *); | |
460 | void lower_complexexpr_stmt (gimple *); | |
7383cb56 | 461 | void lower_bit_query (gimple *); |
a9d6c7fb JJ |
462 | void lower_call (tree, gimple *); |
463 | void lower_asm (gimple *); | |
464 | void lower_stmt (gimple *); | |
465 | ||
466 | /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be | |
467 | merged with their uses. */ | |
468 | bitmap m_names; | |
469 | /* Subset of those for lhs of load statements. These will be | |
470 | cleared in m_names if the loads will be mergeable with all | |
471 | their uses. */ | |
472 | bitmap m_loads; | |
473 | /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive | |
474 | to later passes (arguments or return values of calls). */ | |
475 | bitmap m_preserved; | |
476 | /* Subset of m_names which have a single use. As the lowering | |
477 | can replace various original statements with their lowered | |
478 | form even before it is done iterating over all basic blocks, | |
479 | testing has_single_use for the purpose of emitting clobbers | |
480 | doesn't work properly. */ | |
481 | bitmap m_single_use_names; | |
482 | /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs | |
483 | set in m_names. */ | |
484 | var_map m_map; | |
485 | /* Mapping of the partitions to corresponding decls. */ | |
486 | tree *m_vars; | |
487 | /* Unsigned integer type with limb precision. */ | |
488 | tree m_limb_type; | |
489 | /* Its TYPE_SIZE_UNIT. */ | |
490 | unsigned HOST_WIDE_INT m_limb_size; | |
491 | /* Location of a gimple stmt which is being currently lowered. */ | |
492 | location_t m_loc; | |
493 | /* Current stmt iterator where code is being lowered currently. */ | |
494 | gimple_stmt_iterator m_gsi; | |
495 | /* Statement after which any clobbers should be added if non-NULL. */ | |
496 | gimple *m_after_stmt; | |
497 | /* Set when creating loops to the loop header bb and its preheader. */ | |
498 | basic_block m_bb, m_preheader_bb; | |
499 | /* Stmt iterator after which initialization statements should be emitted. */ | |
500 | gimple_stmt_iterator m_init_gsi; | |
501 | /* Decl into which a mergeable statement stores result. */ | |
502 | tree m_lhs; | |
503 | /* handle_operand/handle_stmt can be invoked in various ways. | |
504 | ||
505 | lower_mergeable_stmt for large _BitInt calls those with constant | |
506 | idx only, expanding to straight line code, for huge _BitInt | |
507 | emits a loop from least significant limb upwards, where each loop | |
508 | iteration handles 2 limbs, plus there can be up to one full limb | |
509 | and one partial limb processed after the loop, where handle_operand | |
510 | and/or handle_stmt are called with constant idx. m_upwards_2limb | |
511 | is set for this case, false otherwise. m_upwards is true if it | |
512 | is either large or huge _BitInt handled by lower_mergeable_stmt, | |
513 | i.e. indexes always increase. | |
514 | ||
515 | Another way is used by lower_comparison_stmt, which walks limbs | |
516 | from most significant to least significant, partial limb if any | |
517 | processed first with constant idx and then loop processing a single | |
518 | limb per iteration with non-constant idx. | |
519 | ||
520 | Another way is used in lower_shift_stmt, where for LSHIFT_EXPR | |
521 | destination limbs are processed from most significant to least | |
522 | significant or for RSHIFT_EXPR the other way around, in loops or | |
523 | straight line code, but idx usually is non-constant (so from | |
524 | handle_operand/handle_stmt POV random access). The LSHIFT_EXPR | |
525 | handling there can access even partial limbs using non-constant | |
526 | idx (then m_var_msb should be true, for all the other cases | |
527 | including lower_mergeable_stmt/lower_comparison_stmt that is | |
528 | not the case and so m_var_msb should be false. | |
529 | ||
530 | m_first should be set the first time handle_operand/handle_stmt | |
531 | is called and clear when it is called for some other limb with | |
532 | the same argument. If the lowering of an operand (e.g. INTEGER_CST) | |
533 | or statement (e.g. +/-/<< with < limb_prec constant) needs some | |
534 | state between the different calls, when m_first is true it should | |
535 | push some trees to m_data vector and also make sure m_data_cnt is | |
536 | incremented by how many trees were pushed, and when m_first is | |
537 | false, it can use the m_data[m_data_cnt] etc. data or update them, | |
538 | just needs to bump m_data_cnt by the same amount as when it was | |
539 | called with m_first set. The toplevel calls to | |
540 | handle_operand/handle_stmt should set m_data_cnt to 0 and truncate | |
541 | m_data vector when setting m_first to true. | |
542 | ||
543 | m_cast_conditional and m_bitfld_load are used when handling a | |
544 | bit-field load inside of a widening cast. handle_cast sometimes | |
545 | needs to do runtime comparisons and handle_operand only conditionally | |
546 | or even in two separate conditional blocks for one idx (once with | |
547 | constant index after comparing the runtime one for equality with the | |
548 | constant). In these cases, m_cast_conditional is set to true and | |
549 | the bit-field load then communicates its m_data_cnt to handle_cast | |
550 | using m_bitfld_load. */ | |
551 | bool m_first; | |
552 | bool m_var_msb; | |
553 | unsigned m_upwards_2limb; | |
554 | bool m_upwards; | |
555 | bool m_cast_conditional; | |
556 | unsigned m_bitfld_load; | |
557 | vec<tree> m_data; | |
558 | unsigned int m_data_cnt; | |
90b98723 | 559 | vec<gimple *> m_returns_twice_calls; |
a9d6c7fb JJ |
560 | }; |
561 | ||
562 | bitint_large_huge::~bitint_large_huge () | |
563 | { | |
564 | BITMAP_FREE (m_names); | |
565 | BITMAP_FREE (m_loads); | |
566 | BITMAP_FREE (m_preserved); | |
567 | BITMAP_FREE (m_single_use_names); | |
568 | if (m_map) | |
569 | delete_var_map (m_map); | |
570 | XDELETEVEC (m_vars); | |
571 | m_data.release (); | |
90b98723 | 572 | m_returns_twice_calls.release (); |
a9d6c7fb JJ |
573 | } |
574 | ||
575 | /* Insert gimple statement G before current location | |
576 | and set its gimple_location. */ | |
577 | ||
578 | void | |
579 | bitint_large_huge::insert_before (gimple *g) | |
580 | { | |
581 | gimple_set_location (g, m_loc); | |
582 | gsi_insert_before (&m_gsi, g, GSI_SAME_STMT); | |
583 | } | |
584 | ||
585 | /* Return type for accessing limb IDX of BITINT_TYPE TYPE. | |
586 | This is normally m_limb_type, except for a partial most | |
587 | significant limb if any. */ | |
588 | ||
589 | tree | |
590 | bitint_large_huge::limb_access_type (tree type, tree idx) | |
591 | { | |
592 | if (type == NULL_TREE) | |
593 | return m_limb_type; | |
594 | unsigned HOST_WIDE_INT i = tree_to_uhwi (idx); | |
595 | unsigned int prec = TYPE_PRECISION (type); | |
596 | gcc_assert (i * limb_prec < prec); | |
597 | if ((i + 1) * limb_prec <= prec) | |
598 | return m_limb_type; | |
599 | else | |
600 | return build_nonstandard_integer_type (prec % limb_prec, | |
601 | TYPE_UNSIGNED (type)); | |
602 | } | |
603 | ||
604 | /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE | |
605 | TYPE. If WRITE_P is true, it will be a store, otherwise a read. */ | |
606 | ||
607 | tree | |
608 | bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p) | |
609 | { | |
610 | tree atype = (tree_fits_uhwi_p (idx) | |
611 | ? limb_access_type (type, idx) : m_limb_type); | |
483c061d JJ |
612 | tree ltype = m_limb_type; |
613 | addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var)); | |
614 | if (as != TYPE_ADDR_SPACE (ltype)) | |
615 | ltype = build_qualified_type (ltype, TYPE_QUALS (ltype) | |
616 | | ENCODE_QUAL_ADDR_SPACE (as)); | |
a9d6c7fb JJ |
617 | tree ret; |
618 | if (DECL_P (var) && tree_fits_uhwi_p (idx)) | |
619 | { | |
620 | tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var))); | |
621 | unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size; | |
483c061d | 622 | ret = build2 (MEM_REF, ltype, |
a9d6c7fb JJ |
623 | build_fold_addr_expr (var), |
624 | build_int_cst (ptype, off)); | |
625 | TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var); | |
626 | TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var); | |
627 | } | |
628 | else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx)) | |
629 | { | |
630 | ret | |
c27148f2 | 631 | = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)), |
a9d6c7fb JJ |
632 | size_binop (PLUS_EXPR, TREE_OPERAND (var, 1), |
633 | build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)), | |
634 | tree_to_uhwi (idx) | |
635 | * m_limb_size))); | |
636 | TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var); | |
637 | TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var); | |
638 | TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var); | |
639 | } | |
640 | else | |
641 | { | |
642 | var = unshare_expr (var); | |
643 | if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE | |
644 | || !useless_type_conversion_p (m_limb_type, | |
645 | TREE_TYPE (TREE_TYPE (var)))) | |
646 | { | |
647 | unsigned HOST_WIDE_INT nelts | |
0613b12d | 648 | = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec); |
483c061d | 649 | tree atype = build_array_type_nelts (ltype, nelts); |
a9d6c7fb JJ |
650 | var = build1 (VIEW_CONVERT_EXPR, atype, var); |
651 | } | |
483c061d | 652 | ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE); |
a9d6c7fb JJ |
653 | } |
654 | if (!write_p && !useless_type_conversion_p (atype, m_limb_type)) | |
655 | { | |
656 | gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret); | |
657 | insert_before (g); | |
658 | ret = gimple_assign_lhs (g); | |
659 | ret = build1 (NOP_EXPR, atype, ret); | |
660 | } | |
661 | return ret; | |
662 | } | |
663 | ||
3e8323e8 JJ |
664 | /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at |
665 | offset BITPOS inside of OBJ. */ | |
666 | ||
667 | tree | |
668 | bitint_large_huge::build_bit_field_ref (tree ftype, tree obj, | |
669 | unsigned HOST_WIDE_INT bitsize, | |
670 | unsigned HOST_WIDE_INT bitpos) | |
671 | { | |
672 | if (INTEGRAL_TYPE_P (TREE_TYPE (obj)) | |
673 | && !type_has_mode_precision_p (TREE_TYPE (obj))) | |
674 | { | |
675 | unsigned HOST_WIDE_INT nelts | |
676 | = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec); | |
677 | tree ltype = m_limb_type; | |
678 | addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj)); | |
679 | if (as != TYPE_ADDR_SPACE (ltype)) | |
680 | ltype = build_qualified_type (ltype, TYPE_QUALS (ltype) | |
681 | | ENCODE_QUAL_ADDR_SPACE (as)); | |
682 | tree atype = build_array_type_nelts (ltype, nelts); | |
683 | obj = build1 (VIEW_CONVERT_EXPR, atype, obj); | |
684 | } | |
685 | return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize), | |
686 | bitsize_int (bitpos)); | |
687 | } | |
688 | ||
a9d6c7fb JJ |
689 | /* Emit a half diamond, |
690 | if (COND) | |
691 | |\ | |
692 | | \ | |
693 | | \ | |
694 | | new_bb1 | |
695 | | / | |
696 | | / | |
697 | |/ | |
698 | or if (COND) new_bb1; | |
699 | PROB is the probability that the condition is true. | |
700 | Updates m_gsi to start of new_bb1. | |
701 | Sets EDGE_TRUE to edge from new_bb1 to successor and | |
702 | EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */ | |
703 | ||
704 | void | |
705 | bitint_large_huge::if_then (gimple *cond, profile_probability prob, | |
706 | edge &edge_true, edge &edge_false) | |
707 | { | |
708 | insert_before (cond); | |
709 | edge e1 = split_block (gsi_bb (m_gsi), cond); | |
710 | edge e2 = split_block (e1->dest, (gimple *) NULL); | |
711 | edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE); | |
712 | e1->flags = EDGE_TRUE_VALUE; | |
713 | e1->probability = prob; | |
714 | e3->probability = prob.invert (); | |
715 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src); | |
716 | edge_true = e2; | |
717 | edge_false = e3; | |
718 | m_gsi = gsi_after_labels (e1->dest); | |
719 | } | |
720 | ||
721 | /* Emit a full diamond, | |
722 | if (COND) | |
723 | /\ | |
724 | / \ | |
725 | / \ | |
726 | new_bb1 new_bb2 | |
727 | \ / | |
728 | \ / | |
729 | \/ | |
730 | or if (COND) new_bb2; else new_bb1; | |
731 | PROB is the probability that the condition is true. | |
732 | Updates m_gsi to start of new_bb2. | |
733 | Sets EDGE_TRUE to edge from new_bb1 to successor and | |
734 | EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb. */ | |
735 | ||
736 | void | |
737 | bitint_large_huge::if_then_else (gimple *cond, profile_probability prob, | |
738 | edge &edge_true, edge &edge_false) | |
739 | { | |
740 | insert_before (cond); | |
741 | edge e1 = split_block (gsi_bb (m_gsi), cond); | |
742 | edge e2 = split_block (e1->dest, (gimple *) NULL); | |
743 | basic_block bb = create_empty_bb (e1->dest); | |
744 | add_bb_to_loop (bb, e1->dest->loop_father); | |
745 | edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE); | |
746 | e1->flags = EDGE_FALSE_VALUE; | |
747 | e3->probability = prob; | |
748 | e1->probability = prob.invert (); | |
3ad9948b | 749 | bb->count = e1->src->count.apply_probability (prob); |
a9d6c7fb JJ |
750 | set_immediate_dominator (CDI_DOMINATORS, bb, e1->src); |
751 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src); | |
3ad9948b | 752 | edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU); |
a9d6c7fb JJ |
753 | edge_false = e2; |
754 | m_gsi = gsi_after_labels (bb); | |
755 | } | |
756 | ||
757 | /* Emit a half diamond with full diamond in it | |
758 | if (COND1) | |
759 | |\ | |
760 | | \ | |
761 | | \ | |
762 | | if (COND2) | |
763 | | / \ | |
764 | | / \ | |
765 | |new_bb1 new_bb2 | |
766 | | | / | |
767 | \ | / | |
768 | \ | / | |
769 | \ | / | |
770 | \|/ | |
771 | or if (COND1) { if (COND2) new_bb2; else new_bb1; } | |
772 | PROB1 is the probability that the condition 1 is true. | |
773 | PROB2 is the probability that the condition 2 is true. | |
774 | Updates m_gsi to start of new_bb1. | |
775 | Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor, | |
776 | EDGE_TRUE_FALSE to edge from new_bb1 to successor and | |
777 | EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb. | |
778 | If COND2 is NULL, this is equivalent to | |
779 | if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE); | |
780 | EDGE_TRUE_TRUE = NULL; */ | |
781 | ||
782 | void | |
783 | bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2, | |
784 | profile_probability prob1, | |
785 | profile_probability prob2, | |
786 | edge &edge_true_true, | |
787 | edge &edge_true_false, | |
788 | edge &edge_false) | |
789 | { | |
790 | edge e2, e3, e4 = NULL; | |
791 | if_then (cond1, prob1, e2, e3); | |
792 | if (cond2 == NULL) | |
793 | { | |
794 | edge_true_true = NULL; | |
795 | edge_true_false = e2; | |
796 | edge_false = e3; | |
797 | return; | |
798 | } | |
799 | insert_before (cond2); | |
800 | e2 = split_block (gsi_bb (m_gsi), cond2); | |
801 | basic_block bb = create_empty_bb (e2->dest); | |
802 | add_bb_to_loop (bb, e2->dest->loop_father); | |
803 | e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE); | |
804 | set_immediate_dominator (CDI_DOMINATORS, bb, e2->src); | |
805 | e4->probability = prob2; | |
806 | e2->flags = EDGE_FALSE_VALUE; | |
807 | e2->probability = prob2.invert (); | |
3ad9948b JJ |
808 | bb->count = e2->src->count.apply_probability (prob2); |
809 | e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU); | |
a9d6c7fb JJ |
810 | e2 = find_edge (e2->dest, e3->dest); |
811 | edge_true_true = e4; | |
812 | edge_true_false = e2; | |
813 | edge_false = e3; | |
814 | m_gsi = gsi_after_labels (e2->src); | |
815 | } | |
816 | ||
817 | /* Emit code to access limb IDX from OP. */ | |
818 | ||
819 | tree | |
820 | bitint_large_huge::handle_operand (tree op, tree idx) | |
821 | { | |
822 | switch (TREE_CODE (op)) | |
823 | { | |
824 | case SSA_NAME: | |
825 | if (m_names == NULL | |
826 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op))) | |
827 | { | |
828 | if (SSA_NAME_IS_DEFAULT_DEF (op)) | |
829 | { | |
830 | if (m_first) | |
831 | { | |
832 | tree v = create_tmp_reg (m_limb_type); | |
833 | if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op))) | |
834 | { | |
835 | DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op)); | |
836 | DECL_SOURCE_LOCATION (v) | |
837 | = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op)); | |
838 | } | |
839 | v = get_or_create_ssa_default_def (cfun, v); | |
840 | m_data.safe_push (v); | |
841 | } | |
842 | tree ret = m_data[m_data_cnt]; | |
843 | m_data_cnt++; | |
844 | if (tree_fits_uhwi_p (idx)) | |
845 | { | |
846 | tree type = limb_access_type (TREE_TYPE (op), idx); | |
847 | ret = add_cast (type, ret); | |
848 | } | |
849 | return ret; | |
850 | } | |
851 | location_t loc_save = m_loc; | |
852 | m_loc = gimple_location (SSA_NAME_DEF_STMT (op)); | |
853 | tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx); | |
854 | m_loc = loc_save; | |
855 | return ret; | |
856 | } | |
857 | int p; | |
858 | gimple *g; | |
859 | tree t; | |
860 | p = var_to_partition (m_map, op); | |
861 | gcc_assert (m_vars[p] != NULL_TREE); | |
862 | t = limb_access (TREE_TYPE (op), m_vars[p], idx, false); | |
863 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t); | |
864 | insert_before (g); | |
865 | t = gimple_assign_lhs (g); | |
866 | if (m_first | |
867 | && m_single_use_names | |
868 | && m_vars[p] != m_lhs | |
869 | && m_after_stmt | |
870 | && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op))) | |
871 | { | |
de072b52 JM |
872 | tree clobber = build_clobber (TREE_TYPE (m_vars[p]), |
873 | CLOBBER_STORAGE_END); | |
a9d6c7fb JJ |
874 | g = gimple_build_assign (m_vars[p], clobber); |
875 | gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt); | |
876 | gsi_insert_after (&gsi, g, GSI_SAME_STMT); | |
877 | } | |
878 | return t; | |
879 | case INTEGER_CST: | |
880 | if (tree_fits_uhwi_p (idx)) | |
881 | { | |
882 | tree c, type = limb_access_type (TREE_TYPE (op), idx); | |
883 | unsigned HOST_WIDE_INT i = tree_to_uhwi (idx); | |
884 | if (m_first) | |
885 | { | |
886 | m_data.safe_push (NULL_TREE); | |
887 | m_data.safe_push (NULL_TREE); | |
888 | } | |
889 | if (limb_prec != HOST_BITS_PER_WIDE_INT) | |
890 | { | |
891 | wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec, | |
892 | TYPE_SIGN (TREE_TYPE (op))); | |
893 | c = wide_int_to_tree (type, | |
894 | wide_int::from (w, TYPE_PRECISION (type), | |
895 | UNSIGNED)); | |
896 | } | |
897 | else if (i >= TREE_INT_CST_EXT_NUNITS (op)) | |
898 | c = build_int_cst (type, | |
899 | tree_int_cst_sgn (op) < 0 ? -1 : 0); | |
900 | else | |
901 | c = build_int_cst (type, TREE_INT_CST_ELT (op, i)); | |
902 | m_data_cnt += 2; | |
903 | return c; | |
904 | } | |
905 | if (m_first | |
906 | || (m_data[m_data_cnt] == NULL_TREE | |
907 | && m_data[m_data_cnt + 1] == NULL_TREE)) | |
908 | { | |
909 | unsigned int prec = TYPE_PRECISION (TREE_TYPE (op)); | |
8a192785 | 910 | unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec); |
a9d6c7fb JJ |
911 | int ext; |
912 | unsigned min_prec = bitint_min_cst_precision (op, ext); | |
913 | if (m_first) | |
914 | { | |
915 | m_data.safe_push (NULL_TREE); | |
916 | m_data.safe_push (NULL_TREE); | |
917 | } | |
918 | if (integer_zerop (op)) | |
919 | { | |
920 | tree c = build_zero_cst (m_limb_type); | |
921 | m_data[m_data_cnt] = c; | |
922 | m_data[m_data_cnt + 1] = c; | |
923 | } | |
924 | else if (integer_all_onesp (op)) | |
925 | { | |
926 | tree c = build_all_ones_cst (m_limb_type); | |
927 | m_data[m_data_cnt] = c; | |
928 | m_data[m_data_cnt + 1] = c; | |
929 | } | |
930 | else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec) | |
931 | { | |
932 | /* Single limb constant. Use a phi with that limb from | |
933 | the preheader edge and 0 or -1 constant from the other edge | |
934 | and for the second limb in the loop. */ | |
935 | tree out; | |
936 | gcc_assert (m_first); | |
937 | m_data.pop (); | |
938 | m_data.pop (); | |
07efd566 JJ |
939 | prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out, |
940 | build_int_cst (m_limb_type, ext)); | |
a9d6c7fb JJ |
941 | } |
942 | else if (min_prec > prec - rem - 2 * limb_prec) | |
943 | { | |
944 | /* Constant which has enough significant bits that it isn't | |
945 | worth trying to save .rodata space by extending from smaller | |
946 | number. */ | |
947 | tree type; | |
948 | if (m_var_msb) | |
949 | type = TREE_TYPE (op); | |
950 | else | |
951 | /* If we have a guarantee the most significant partial limb | |
952 | (if any) will be only accessed through handle_operand | |
953 | with INTEGER_CST idx, we don't need to include the partial | |
954 | limb in .rodata. */ | |
955 | type = build_bitint_type (prec - rem, 1); | |
956 | tree c = tree_output_constant_def (fold_convert (type, op)); | |
957 | m_data[m_data_cnt] = c; | |
958 | m_data[m_data_cnt + 1] = NULL_TREE; | |
959 | } | |
960 | else if (m_upwards_2limb) | |
961 | { | |
962 | /* Constant with smaller number of bits. Trade conditional | |
963 | code for .rodata space by extending from smaller number. */ | |
964 | min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec); | |
965 | tree type = build_bitint_type (min_prec, 1); | |
966 | tree c = tree_output_constant_def (fold_convert (type, op)); | |
967 | tree idx2 = make_ssa_name (sizetype); | |
968 | g = gimple_build_assign (idx2, PLUS_EXPR, idx, size_one_node); | |
969 | insert_before (g); | |
970 | g = gimple_build_cond (LT_EXPR, idx, | |
971 | size_int (min_prec / limb_prec), | |
972 | NULL_TREE, NULL_TREE); | |
973 | edge edge_true, edge_false; | |
974 | if_then (g, (min_prec >= (prec - rem) / 2 | |
975 | ? profile_probability::likely () | |
976 | : profile_probability::unlikely ()), | |
977 | edge_true, edge_false); | |
978 | tree c1 = limb_access (TREE_TYPE (op), c, idx, false); | |
979 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1); | |
980 | insert_before (g); | |
981 | c1 = gimple_assign_lhs (g); | |
982 | tree c2 = limb_access (TREE_TYPE (op), c, idx2, false); | |
983 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2); | |
984 | insert_before (g); | |
985 | c2 = gimple_assign_lhs (g); | |
986 | tree c3 = build_int_cst (m_limb_type, ext); | |
987 | m_gsi = gsi_after_labels (edge_true->dest); | |
988 | m_data[m_data_cnt] = make_ssa_name (m_limb_type); | |
989 | m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type); | |
990 | gphi *phi = create_phi_node (m_data[m_data_cnt], | |
991 | edge_true->dest); | |
992 | add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION); | |
993 | add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION); | |
994 | phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest); | |
995 | add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION); | |
996 | add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION); | |
997 | } | |
998 | else | |
999 | { | |
1000 | /* Constant with smaller number of bits. Trade conditional | |
1001 | code for .rodata space by extending from smaller number. | |
1002 | Version for loops with random access to the limbs or | |
1003 | downwards loops. */ | |
1004 | min_prec = CEIL (min_prec, limb_prec) * limb_prec; | |
1005 | tree c; | |
1006 | if (min_prec <= (unsigned) limb_prec) | |
1007 | c = fold_convert (m_limb_type, op); | |
1008 | else | |
1009 | { | |
1010 | tree type = build_bitint_type (min_prec, 1); | |
1011 | c = tree_output_constant_def (fold_convert (type, op)); | |
1012 | } | |
1013 | m_data[m_data_cnt] = c; | |
1014 | m_data[m_data_cnt + 1] = integer_type_node; | |
1015 | } | |
1016 | t = m_data[m_data_cnt]; | |
1017 | if (m_data[m_data_cnt + 1] == NULL_TREE) | |
1018 | { | |
1019 | t = limb_access (TREE_TYPE (op), t, idx, false); | |
1020 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t); | |
1021 | insert_before (g); | |
1022 | t = gimple_assign_lhs (g); | |
1023 | } | |
1024 | } | |
1025 | else if (m_data[m_data_cnt + 1] == NULL_TREE) | |
1026 | { | |
1027 | t = limb_access (TREE_TYPE (op), m_data[m_data_cnt], idx, false); | |
1028 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t); | |
1029 | insert_before (g); | |
1030 | t = gimple_assign_lhs (g); | |
1031 | } | |
1032 | else | |
1033 | t = m_data[m_data_cnt + 1]; | |
1034 | if (m_data[m_data_cnt + 1] == integer_type_node) | |
1035 | { | |
1036 | unsigned int prec = TYPE_PRECISION (TREE_TYPE (op)); | |
8a192785 | 1037 | unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec); |
c7557916 | 1038 | int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0; |
a9d6c7fb JJ |
1039 | tree c = m_data[m_data_cnt]; |
1040 | unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c)); | |
1041 | g = gimple_build_cond (LT_EXPR, idx, | |
1042 | size_int (min_prec / limb_prec), | |
1043 | NULL_TREE, NULL_TREE); | |
1044 | edge edge_true, edge_false; | |
1045 | if_then (g, (min_prec >= (prec - rem) / 2 | |
1046 | ? profile_probability::likely () | |
1047 | : profile_probability::unlikely ()), | |
1048 | edge_true, edge_false); | |
1049 | if (min_prec > (unsigned) limb_prec) | |
1050 | { | |
1051 | c = limb_access (TREE_TYPE (op), c, idx, false); | |
1052 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c); | |
1053 | insert_before (g); | |
1054 | c = gimple_assign_lhs (g); | |
1055 | } | |
1056 | tree c2 = build_int_cst (m_limb_type, ext); | |
1057 | m_gsi = gsi_after_labels (edge_true->dest); | |
1058 | t = make_ssa_name (m_limb_type); | |
1059 | gphi *phi = create_phi_node (t, edge_true->dest); | |
1060 | add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION); | |
1061 | add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION); | |
1062 | } | |
1063 | m_data_cnt += 2; | |
1064 | return t; | |
1065 | default: | |
1066 | gcc_unreachable (); | |
1067 | } | |
1068 | } | |
1069 | ||
1070 | /* Helper method, add a PHI node with VAL from preheader edge if | |
1071 | inside of a loop and m_first. Keep state in a pair of m_data | |
07efd566 JJ |
1072 | elements. If VAL_OUT is non-NULL, use that as PHI argument from |
1073 | the latch edge, otherwise create a new SSA_NAME for it and let | |
1074 | caller initialize it. */ | |
a9d6c7fb JJ |
1075 | |
1076 | tree | |
07efd566 JJ |
1077 | bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out, |
1078 | tree val_out) | |
a9d6c7fb JJ |
1079 | { |
1080 | if (!m_first) | |
1081 | { | |
1082 | *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1]; | |
1083 | return m_data[m_data_cnt]; | |
1084 | } | |
1085 | ||
1086 | *data_out = NULL_TREE; | |
1087 | if (tree_fits_uhwi_p (idx)) | |
1088 | { | |
1089 | m_data.safe_push (val); | |
1090 | m_data.safe_push (NULL_TREE); | |
1091 | return val; | |
1092 | } | |
1093 | ||
1094 | tree in = make_ssa_name (TREE_TYPE (val)); | |
1095 | gphi *phi = create_phi_node (in, m_bb); | |
1096 | edge e1 = find_edge (m_preheader_bb, m_bb); | |
1097 | edge e2 = EDGE_PRED (m_bb, 0); | |
1098 | if (e1 == e2) | |
1099 | e2 = EDGE_PRED (m_bb, 1); | |
1100 | add_phi_arg (phi, val, e1, UNKNOWN_LOCATION); | |
07efd566 | 1101 | tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val)); |
a9d6c7fb JJ |
1102 | add_phi_arg (phi, out, e2, UNKNOWN_LOCATION); |
1103 | m_data.safe_push (in); | |
1104 | m_data.safe_push (out); | |
1105 | return in; | |
1106 | } | |
1107 | ||
1108 | /* Return VAL cast to TYPE. If VAL is INTEGER_CST, just | |
1109 | convert it without emitting any code, otherwise emit | |
1110 | the conversion statement before the current location. */ | |
1111 | ||
1112 | tree | |
1113 | bitint_large_huge::add_cast (tree type, tree val) | |
1114 | { | |
1115 | if (TREE_CODE (val) == INTEGER_CST) | |
1116 | return fold_convert (type, val); | |
1117 | ||
1118 | tree lhs = make_ssa_name (type); | |
1119 | gimple *g = gimple_build_assign (lhs, NOP_EXPR, val); | |
1120 | insert_before (g); | |
1121 | return lhs; | |
1122 | } | |
1123 | ||
1124 | /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR. */ | |
1125 | ||
1126 | tree | |
1127 | bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2, | |
1128 | tree idx) | |
1129 | { | |
1130 | tree lhs, data_out, ctype; | |
1131 | tree rhs1_type = TREE_TYPE (rhs1); | |
1132 | gimple *g; | |
1133 | tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, | |
1134 | &data_out); | |
1135 | ||
1136 | if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab, | |
1137 | TYPE_MODE (m_limb_type)) != CODE_FOR_nothing) | |
1138 | { | |
1139 | ctype = build_complex_type (m_limb_type); | |
1140 | if (!types_compatible_p (rhs1_type, m_limb_type)) | |
1141 | { | |
1142 | if (!TYPE_UNSIGNED (rhs1_type)) | |
1143 | { | |
1144 | tree type = unsigned_type_for (rhs1_type); | |
1145 | rhs1 = add_cast (type, rhs1); | |
1146 | rhs2 = add_cast (type, rhs2); | |
1147 | } | |
1148 | rhs1 = add_cast (m_limb_type, rhs1); | |
1149 | rhs2 = add_cast (m_limb_type, rhs2); | |
1150 | } | |
1151 | lhs = make_ssa_name (ctype); | |
1152 | g = gimple_build_call_internal (code == PLUS_EXPR | |
1153 | ? IFN_UADDC : IFN_USUBC, | |
1154 | 3, rhs1, rhs2, data_in); | |
1155 | gimple_call_set_lhs (g, lhs); | |
1156 | insert_before (g); | |
1157 | if (data_out == NULL_TREE) | |
1158 | data_out = make_ssa_name (m_limb_type); | |
1159 | g = gimple_build_assign (data_out, IMAGPART_EXPR, | |
1160 | build1 (IMAGPART_EXPR, m_limb_type, lhs)); | |
1161 | insert_before (g); | |
1162 | } | |
1163 | else if (types_compatible_p (rhs1_type, m_limb_type)) | |
1164 | { | |
1165 | ctype = build_complex_type (m_limb_type); | |
1166 | lhs = make_ssa_name (ctype); | |
1167 | g = gimple_build_call_internal (code == PLUS_EXPR | |
1168 | ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW, | |
1169 | 2, rhs1, rhs2); | |
1170 | gimple_call_set_lhs (g, lhs); | |
1171 | insert_before (g); | |
1172 | if (data_out == NULL_TREE) | |
1173 | data_out = make_ssa_name (m_limb_type); | |
1174 | if (!integer_zerop (data_in)) | |
1175 | { | |
1176 | rhs1 = make_ssa_name (m_limb_type); | |
1177 | g = gimple_build_assign (rhs1, REALPART_EXPR, | |
1178 | build1 (REALPART_EXPR, m_limb_type, lhs)); | |
1179 | insert_before (g); | |
1180 | rhs2 = make_ssa_name (m_limb_type); | |
1181 | g = gimple_build_assign (rhs2, IMAGPART_EXPR, | |
1182 | build1 (IMAGPART_EXPR, m_limb_type, lhs)); | |
1183 | insert_before (g); | |
1184 | lhs = make_ssa_name (ctype); | |
1185 | g = gimple_build_call_internal (code == PLUS_EXPR | |
1186 | ? IFN_ADD_OVERFLOW | |
1187 | : IFN_SUB_OVERFLOW, | |
1188 | 2, rhs1, data_in); | |
1189 | gimple_call_set_lhs (g, lhs); | |
1190 | insert_before (g); | |
1191 | data_in = make_ssa_name (m_limb_type); | |
1192 | g = gimple_build_assign (data_in, IMAGPART_EXPR, | |
1193 | build1 (IMAGPART_EXPR, m_limb_type, lhs)); | |
1194 | insert_before (g); | |
1195 | g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in); | |
1196 | insert_before (g); | |
1197 | } | |
1198 | else | |
1199 | { | |
1200 | g = gimple_build_assign (data_out, IMAGPART_EXPR, | |
1201 | build1 (IMAGPART_EXPR, m_limb_type, lhs)); | |
1202 | insert_before (g); | |
1203 | } | |
1204 | } | |
1205 | else | |
1206 | { | |
1207 | tree in = add_cast (rhs1_type, data_in); | |
1208 | lhs = make_ssa_name (rhs1_type); | |
1209 | g = gimple_build_assign (lhs, code, rhs1, rhs2); | |
1210 | insert_before (g); | |
1211 | rhs1 = make_ssa_name (rhs1_type); | |
1212 | g = gimple_build_assign (rhs1, code, lhs, in); | |
1213 | insert_before (g); | |
1214 | m_data[m_data_cnt] = NULL_TREE; | |
1215 | m_data_cnt += 2; | |
1216 | return rhs1; | |
1217 | } | |
1218 | rhs1 = make_ssa_name (m_limb_type); | |
1219 | g = gimple_build_assign (rhs1, REALPART_EXPR, | |
1220 | build1 (REALPART_EXPR, m_limb_type, lhs)); | |
1221 | insert_before (g); | |
1222 | if (!types_compatible_p (rhs1_type, m_limb_type)) | |
1223 | rhs1 = add_cast (rhs1_type, rhs1); | |
1224 | m_data[m_data_cnt] = data_out; | |
1225 | m_data_cnt += 2; | |
1226 | return rhs1; | |
1227 | } | |
1228 | ||
1229 | /* Helper function for handle_stmt method, handle LSHIFT_EXPR by | |
1230 | count in [0, limb_prec - 1] range. */ | |
1231 | ||
1232 | tree | |
1233 | bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx) | |
1234 | { | |
1235 | unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2); | |
1236 | gcc_checking_assert (cnt < (unsigned) limb_prec); | |
1237 | if (cnt == 0) | |
1238 | return rhs1; | |
1239 | ||
1240 | tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1); | |
1241 | gimple *g; | |
1242 | tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, | |
1243 | &data_out); | |
1244 | ||
1245 | if (!integer_zerop (data_in)) | |
1246 | { | |
1247 | lhs = make_ssa_name (m_limb_type); | |
1248 | g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in, | |
1249 | build_int_cst (unsigned_type_node, | |
1250 | limb_prec - cnt)); | |
1251 | insert_before (g); | |
1252 | if (!types_compatible_p (rhs1_type, m_limb_type)) | |
1253 | lhs = add_cast (rhs1_type, lhs); | |
1254 | data_in = lhs; | |
1255 | } | |
1256 | if (types_compatible_p (rhs1_type, m_limb_type)) | |
1257 | { | |
1258 | if (data_out == NULL_TREE) | |
1259 | data_out = make_ssa_name (m_limb_type); | |
1260 | g = gimple_build_assign (data_out, rhs1); | |
1261 | insert_before (g); | |
1262 | } | |
1263 | if (cnt < (unsigned) TYPE_PRECISION (rhs1_type)) | |
1264 | { | |
1265 | lhs = make_ssa_name (rhs1_type); | |
1266 | g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2); | |
1267 | insert_before (g); | |
1268 | if (!integer_zerop (data_in)) | |
1269 | { | |
1270 | rhs1 = lhs; | |
1271 | lhs = make_ssa_name (rhs1_type); | |
1272 | g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in); | |
1273 | insert_before (g); | |
1274 | } | |
1275 | } | |
1276 | else | |
1277 | lhs = data_in; | |
1278 | m_data[m_data_cnt] = data_out; | |
1279 | m_data_cnt += 2; | |
1280 | return lhs; | |
1281 | } | |
1282 | ||
1283 | /* Helper function for handle_stmt method, handle an integral | |
1284 | to integral conversion. */ | |
1285 | ||
1286 | tree | |
1287 | bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx) | |
1288 | { | |
1289 | tree rhs_type = TREE_TYPE (rhs1); | |
1290 | gimple *g; | |
efc677f3 | 1291 | if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST) |
a9d6c7fb JJ |
1292 | && TREE_CODE (lhs_type) == BITINT_TYPE |
1293 | && TREE_CODE (rhs_type) == BITINT_TYPE | |
1294 | && bitint_precision_kind (lhs_type) >= bitint_prec_large | |
1295 | && bitint_precision_kind (rhs_type) >= bitint_prec_large) | |
1296 | { | |
1297 | if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type) | |
1298 | /* If lhs has bigger precision than rhs, we can use | |
1299 | the simple case only if there is a guarantee that | |
1300 | the most significant limb is handled in straight | |
1301 | line code. If m_var_msb (on left shifts) or | |
1302 | if m_upwards_2limb * limb_prec is equal to | |
9511b91c JJ |
1303 | lhs precision or if not m_upwards_2limb and lhs_type |
1304 | has precision which is multiple of limb_prec that is | |
1305 | not the case. */ | |
a9d6c7fb | 1306 | || (!m_var_msb |
4ff5e7ce JJ |
1307 | && (CEIL (TYPE_PRECISION (lhs_type), limb_prec) |
1308 | == CEIL (TYPE_PRECISION (rhs_type), limb_prec)) | |
9511b91c JJ |
1309 | && ((!m_upwards_2limb |
1310 | && (TYPE_PRECISION (lhs_type) % limb_prec != 0)) | |
1311 | || (m_upwards_2limb | |
1312 | && (m_upwards_2limb * limb_prec | |
1313 | < TYPE_PRECISION (lhs_type)))))) | |
a9d6c7fb JJ |
1314 | { |
1315 | rhs1 = handle_operand (rhs1, idx); | |
1316 | if (tree_fits_uhwi_p (idx)) | |
1317 | { | |
1318 | tree type = limb_access_type (lhs_type, idx); | |
1319 | if (!types_compatible_p (type, TREE_TYPE (rhs1))) | |
1320 | rhs1 = add_cast (type, rhs1); | |
1321 | } | |
1322 | return rhs1; | |
1323 | } | |
1324 | tree t; | |
1325 | /* Indexes lower than this don't need any special processing. */ | |
1326 | unsigned low = ((unsigned) TYPE_PRECISION (rhs_type) | |
1327 | - !TYPE_UNSIGNED (rhs_type)) / limb_prec; | |
1328 | /* Indexes >= than this always contain an extension. */ | |
1329 | unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec); | |
1330 | bool save_first = m_first; | |
1331 | if (m_first) | |
1332 | { | |
1333 | m_data.safe_push (NULL_TREE); | |
1334 | m_data.safe_push (NULL_TREE); | |
1335 | m_data.safe_push (NULL_TREE); | |
1336 | if (TYPE_UNSIGNED (rhs_type)) | |
1337 | /* No need to keep state between iterations. */ | |
1338 | ; | |
1339 | else if (m_upwards && !m_upwards_2limb) | |
1340 | /* We need to keep state between iterations, but | |
1341 | not within any loop, everything is straight line | |
1342 | code with only increasing indexes. */ | |
1343 | ; | |
1344 | else if (!m_upwards_2limb) | |
1345 | { | |
1346 | unsigned save_data_cnt = m_data_cnt; | |
1347 | gimple_stmt_iterator save_gsi = m_gsi; | |
1348 | m_gsi = m_init_gsi; | |
1349 | if (gsi_end_p (m_gsi)) | |
1350 | m_gsi = gsi_after_labels (gsi_bb (m_gsi)); | |
1351 | else | |
1352 | gsi_next (&m_gsi); | |
1353 | m_data_cnt = save_data_cnt + 3; | |
1354 | t = handle_operand (rhs1, size_int (low)); | |
1355 | m_first = false; | |
1356 | m_data[save_data_cnt + 2] | |
1357 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1358 | m_data_cnt = save_data_cnt; | |
1359 | t = add_cast (signed_type_for (m_limb_type), t); | |
1360 | tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1); | |
1361 | tree n = make_ssa_name (TREE_TYPE (t)); | |
1362 | g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1); | |
1363 | insert_before (g); | |
1364 | m_data[save_data_cnt + 1] = add_cast (m_limb_type, n); | |
9a96a9e4 JJ |
1365 | m_init_gsi = m_gsi; |
1366 | if (gsi_end_p (m_init_gsi)) | |
1367 | m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi)); | |
1368 | else | |
1369 | gsi_prev (&m_init_gsi); | |
a9d6c7fb JJ |
1370 | m_gsi = save_gsi; |
1371 | } | |
1372 | else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type)) | |
1373 | /* We need to keep state between iterations, but | |
1374 | fortunately not within the loop, only afterwards. */ | |
1375 | ; | |
1376 | else | |
1377 | { | |
1378 | tree out; | |
1379 | m_data.truncate (m_data_cnt); | |
1380 | prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out); | |
1381 | m_data.safe_push (NULL_TREE); | |
1382 | } | |
1383 | } | |
1384 | ||
1385 | unsigned save_data_cnt = m_data_cnt; | |
1386 | m_data_cnt += 3; | |
1387 | if (!tree_fits_uhwi_p (idx)) | |
1388 | { | |
1389 | if (m_upwards_2limb | |
97e49bf0 | 1390 | && low >= m_upwards_2limb - m_first) |
a9d6c7fb JJ |
1391 | { |
1392 | rhs1 = handle_operand (rhs1, idx); | |
1393 | if (m_first) | |
1394 | m_data[save_data_cnt + 2] | |
1395 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1396 | m_first = save_first; | |
1397 | return rhs1; | |
1398 | } | |
1399 | bool single_comparison | |
1400 | = low == high || (m_upwards_2limb && (low & 1) == m_first); | |
97e49bf0 JJ |
1401 | tree idxc = idx; |
1402 | if (!single_comparison | |
1403 | && m_upwards_2limb | |
1404 | && !m_first | |
1405 | && low + 1 == m_upwards_2limb) | |
1406 | /* In this case we know that idx <= low always, | |
1407 | so effectively we just needs a single comparison, | |
1408 | idx < low or idx == low, but we'd need to emit different | |
1409 | code for the 2 branches than single_comparison normally | |
1410 | emits. So, instead of special-casing that, emit a | |
1411 | low <= low comparison which cfg cleanup will clean up | |
1412 | at the end of the pass. */ | |
1413 | idxc = size_int (low); | |
a9d6c7fb | 1414 | g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR, |
97e49bf0 | 1415 | idxc, size_int (low), NULL_TREE, NULL_TREE); |
a9d6c7fb JJ |
1416 | edge edge_true_true, edge_true_false, edge_false; |
1417 | if_then_if_then_else (g, (single_comparison ? NULL | |
1418 | : gimple_build_cond (EQ_EXPR, idx, | |
1419 | size_int (low), | |
1420 | NULL_TREE, | |
1421 | NULL_TREE)), | |
1422 | profile_probability::likely (), | |
1423 | profile_probability::unlikely (), | |
1424 | edge_true_true, edge_true_false, edge_false); | |
1425 | bool save_cast_conditional = m_cast_conditional; | |
1426 | m_cast_conditional = true; | |
1427 | m_bitfld_load = 0; | |
1428 | tree t1 = handle_operand (rhs1, idx), t2 = NULL_TREE; | |
1429 | if (m_first) | |
1430 | m_data[save_data_cnt + 2] | |
1431 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1432 | tree ext = NULL_TREE; | |
1433 | tree bitfld = NULL_TREE; | |
1434 | if (!single_comparison) | |
1435 | { | |
1436 | m_gsi = gsi_after_labels (edge_true_true->src); | |
1437 | m_first = false; | |
1438 | m_data_cnt = save_data_cnt + 3; | |
1439 | if (m_bitfld_load) | |
1440 | { | |
1441 | bitfld = m_data[m_bitfld_load]; | |
1442 | m_data[m_bitfld_load] = m_data[m_bitfld_load + 2]; | |
1443 | m_bitfld_load = 0; | |
1444 | } | |
1445 | t2 = handle_operand (rhs1, size_int (low)); | |
1446 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2))) | |
1447 | t2 = add_cast (m_limb_type, t2); | |
1448 | if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb) | |
1449 | { | |
1450 | ext = add_cast (signed_type_for (m_limb_type), t2); | |
1451 | tree lpm1 = build_int_cst (unsigned_type_node, | |
1452 | limb_prec - 1); | |
1453 | tree n = make_ssa_name (TREE_TYPE (ext)); | |
1454 | g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1); | |
1455 | insert_before (g); | |
1456 | ext = add_cast (m_limb_type, n); | |
1457 | } | |
1458 | } | |
1459 | tree t3; | |
1460 | if (TYPE_UNSIGNED (rhs_type)) | |
1461 | t3 = build_zero_cst (m_limb_type); | |
1462 | else if (m_upwards_2limb && (save_first || ext != NULL_TREE)) | |
1463 | t3 = m_data[save_data_cnt]; | |
1464 | else | |
1465 | t3 = m_data[save_data_cnt + 1]; | |
1466 | m_gsi = gsi_after_labels (edge_true_false->dest); | |
1467 | t = make_ssa_name (m_limb_type); | |
1468 | gphi *phi = create_phi_node (t, edge_true_false->dest); | |
1469 | add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION); | |
1470 | add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION); | |
1471 | if (edge_true_true) | |
1472 | add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION); | |
1473 | if (ext) | |
1474 | { | |
1475 | tree t4 = make_ssa_name (m_limb_type); | |
1476 | phi = create_phi_node (t4, edge_true_false->dest); | |
1477 | add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false, | |
1478 | UNKNOWN_LOCATION); | |
1479 | add_phi_arg (phi, m_data[save_data_cnt], edge_false, | |
1480 | UNKNOWN_LOCATION); | |
1481 | add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION); | |
be9e8de6 JJ |
1482 | if (!save_cast_conditional) |
1483 | { | |
1484 | g = gimple_build_assign (m_data[save_data_cnt + 1], t4); | |
1485 | insert_before (g); | |
1486 | } | |
1487 | else | |
1488 | for (basic_block bb = gsi_bb (m_gsi);;) | |
1489 | { | |
1490 | edge e1 = single_succ_edge (bb); | |
1491 | edge e2 = find_edge (e1->dest, m_bb), e3; | |
1492 | tree t5 = (e2 ? m_data[save_data_cnt + 1] | |
1493 | : make_ssa_name (m_limb_type)); | |
1494 | phi = create_phi_node (t5, e1->dest); | |
1495 | edge_iterator ei; | |
1496 | FOR_EACH_EDGE (e3, ei, e1->dest->preds) | |
1497 | add_phi_arg (phi, (e3 == e1 ? t4 | |
1498 | : build_zero_cst (m_limb_type)), | |
1499 | e3, UNKNOWN_LOCATION); | |
1500 | if (e2) | |
1501 | break; | |
1502 | t4 = t5; | |
1503 | bb = e1->dest; | |
1504 | } | |
a9d6c7fb JJ |
1505 | } |
1506 | if (m_bitfld_load) | |
1507 | { | |
1508 | tree t4; | |
48530efd | 1509 | if (!save_first && !save_cast_conditional) |
a9d6c7fb JJ |
1510 | t4 = m_data[m_bitfld_load + 1]; |
1511 | else | |
1512 | t4 = make_ssa_name (m_limb_type); | |
1513 | phi = create_phi_node (t4, edge_true_false->dest); | |
1514 | add_phi_arg (phi, | |
1515 | edge_true_true ? bitfld : m_data[m_bitfld_load], | |
1516 | edge_true_false, UNKNOWN_LOCATION); | |
1517 | add_phi_arg (phi, m_data[m_bitfld_load + 2], | |
1518 | edge_false, UNKNOWN_LOCATION); | |
1519 | if (edge_true_true) | |
1520 | add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true, | |
1521 | UNKNOWN_LOCATION); | |
48530efd JJ |
1522 | if (save_cast_conditional) |
1523 | for (basic_block bb = gsi_bb (m_gsi);;) | |
1524 | { | |
1525 | edge e1 = single_succ_edge (bb); | |
1526 | edge e2 = find_edge (e1->dest, m_bb), e3; | |
1527 | tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1] | |
1528 | : make_ssa_name (m_limb_type)); | |
1529 | phi = create_phi_node (t5, e1->dest); | |
1530 | edge_iterator ei; | |
1531 | FOR_EACH_EDGE (e3, ei, e1->dest->preds) | |
1532 | add_phi_arg (phi, (e3 == e1 ? t4 | |
1533 | : build_zero_cst (m_limb_type)), | |
1534 | e3, UNKNOWN_LOCATION); | |
1535 | t4 = t5; | |
1536 | if (e2) | |
1537 | break; | |
1538 | bb = e1->dest; | |
1539 | } | |
a9d6c7fb JJ |
1540 | m_data[m_bitfld_load] = t4; |
1541 | m_data[m_bitfld_load + 2] = t4; | |
1542 | m_bitfld_load = 0; | |
1543 | } | |
1544 | m_cast_conditional = save_cast_conditional; | |
1545 | m_first = save_first; | |
1546 | return t; | |
1547 | } | |
1548 | else | |
1549 | { | |
1550 | if (tree_to_uhwi (idx) < low) | |
1551 | { | |
1552 | t = handle_operand (rhs1, idx); | |
1553 | if (m_first) | |
1554 | m_data[save_data_cnt + 2] | |
1555 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1556 | } | |
1557 | else if (tree_to_uhwi (idx) < high) | |
1558 | { | |
1559 | t = handle_operand (rhs1, size_int (low)); | |
1560 | if (m_first) | |
1561 | m_data[save_data_cnt + 2] | |
1562 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1563 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t))) | |
1564 | t = add_cast (m_limb_type, t); | |
1565 | tree ext = NULL_TREE; | |
1566 | if (!TYPE_UNSIGNED (rhs_type) && m_upwards) | |
1567 | { | |
1568 | ext = add_cast (signed_type_for (m_limb_type), t); | |
1569 | tree lpm1 = build_int_cst (unsigned_type_node, | |
1570 | limb_prec - 1); | |
1571 | tree n = make_ssa_name (TREE_TYPE (ext)); | |
1572 | g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1); | |
1573 | insert_before (g); | |
1574 | ext = add_cast (m_limb_type, n); | |
1575 | m_data[save_data_cnt + 1] = ext; | |
1576 | } | |
1577 | } | |
1578 | else | |
1579 | { | |
1580 | if (TYPE_UNSIGNED (rhs_type) && m_first) | |
1581 | { | |
1582 | handle_operand (rhs1, size_zero_node); | |
1583 | m_data[save_data_cnt + 2] | |
1584 | = build_int_cst (NULL_TREE, m_data_cnt); | |
1585 | } | |
1586 | else | |
1587 | m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]); | |
1588 | if (TYPE_UNSIGNED (rhs_type)) | |
1589 | t = build_zero_cst (m_limb_type); | |
d3defa43 | 1590 | else if (m_bb && m_data[save_data_cnt]) |
be9e8de6 | 1591 | t = m_data[save_data_cnt]; |
a9d6c7fb JJ |
1592 | else |
1593 | t = m_data[save_data_cnt + 1]; | |
1594 | } | |
1595 | tree type = limb_access_type (lhs_type, idx); | |
1596 | if (!useless_type_conversion_p (type, m_limb_type)) | |
1597 | t = add_cast (type, t); | |
1598 | m_first = save_first; | |
1599 | return t; | |
1600 | } | |
1601 | } | |
1602 | else if (TREE_CODE (lhs_type) == BITINT_TYPE | |
1603 | && bitint_precision_kind (lhs_type) >= bitint_prec_large | |
1604 | && INTEGRAL_TYPE_P (rhs_type)) | |
1605 | { | |
1606 | /* Add support for 3 or more limbs filled in from normal integral | |
1607 | type if this assert fails. If no target chooses limb mode smaller | |
1608 | than half of largest supported normal integral type, this will not | |
1609 | be needed. */ | |
1610 | gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec); | |
1611 | tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE; | |
1612 | if (m_first) | |
1613 | { | |
1614 | gimple_stmt_iterator save_gsi = m_gsi; | |
1615 | m_gsi = m_init_gsi; | |
1616 | if (gsi_end_p (m_gsi)) | |
1617 | m_gsi = gsi_after_labels (gsi_bb (m_gsi)); | |
1618 | else | |
1619 | gsi_next (&m_gsi); | |
1620 | if (TREE_CODE (rhs_type) == BITINT_TYPE | |
1621 | && bitint_precision_kind (rhs_type) == bitint_prec_middle) | |
1622 | { | |
1623 | tree type = NULL_TREE; | |
1624 | rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type); | |
1625 | rhs_type = TREE_TYPE (rhs1); | |
1626 | } | |
1627 | r1 = rhs1; | |
1628 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1))) | |
1629 | r1 = add_cast (m_limb_type, rhs1); | |
1630 | if (TYPE_PRECISION (rhs_type) > limb_prec) | |
1631 | { | |
1632 | g = gimple_build_assign (make_ssa_name (rhs_type), | |
1633 | RSHIFT_EXPR, rhs1, | |
1634 | build_int_cst (unsigned_type_node, | |
1635 | limb_prec)); | |
1636 | insert_before (g); | |
1637 | r2 = add_cast (m_limb_type, gimple_assign_lhs (g)); | |
1638 | } | |
1639 | if (TYPE_UNSIGNED (rhs_type)) | |
1640 | rext = build_zero_cst (m_limb_type); | |
1641 | else | |
1642 | { | |
1643 | rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1); | |
1644 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)), | |
1645 | RSHIFT_EXPR, rext, | |
1646 | build_int_cst (unsigned_type_node, | |
1647 | limb_prec - 1)); | |
1648 | insert_before (g); | |
1649 | rext = add_cast (m_limb_type, gimple_assign_lhs (g)); | |
1650 | } | |
9a96a9e4 JJ |
1651 | m_init_gsi = m_gsi; |
1652 | if (gsi_end_p (m_init_gsi)) | |
1653 | m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi)); | |
1654 | else | |
1655 | gsi_prev (&m_init_gsi); | |
a9d6c7fb JJ |
1656 | m_gsi = save_gsi; |
1657 | } | |
1658 | tree t; | |
1659 | if (m_upwards_2limb) | |
1660 | { | |
1661 | if (m_first) | |
1662 | { | |
1663 | tree out1, out2; | |
07efd566 | 1664 | prepare_data_in_out (r1, idx, &out1, rext); |
a9d6c7fb JJ |
1665 | if (TYPE_PRECISION (rhs_type) > limb_prec) |
1666 | { | |
07efd566 | 1667 | prepare_data_in_out (r2, idx, &out2, rext); |
a9d6c7fb JJ |
1668 | m_data.pop (); |
1669 | t = m_data.pop (); | |
1670 | m_data[m_data_cnt + 1] = t; | |
1671 | } | |
1672 | else | |
1673 | m_data[m_data_cnt + 1] = rext; | |
1674 | m_data.safe_push (rext); | |
1675 | t = m_data[m_data_cnt]; | |
1676 | } | |
1677 | else if (!tree_fits_uhwi_p (idx)) | |
1678 | t = m_data[m_data_cnt + 1]; | |
1679 | else | |
1680 | { | |
1681 | tree type = limb_access_type (lhs_type, idx); | |
1682 | t = m_data[m_data_cnt + 2]; | |
1683 | if (!useless_type_conversion_p (type, m_limb_type)) | |
1684 | t = add_cast (type, t); | |
1685 | } | |
1686 | m_data_cnt += 3; | |
1687 | return t; | |
1688 | } | |
1689 | else if (m_first) | |
1690 | { | |
1691 | m_data.safe_push (r1); | |
1692 | m_data.safe_push (r2); | |
1693 | m_data.safe_push (rext); | |
1694 | } | |
1695 | if (tree_fits_uhwi_p (idx)) | |
1696 | { | |
1697 | tree type = limb_access_type (lhs_type, idx); | |
1698 | if (integer_zerop (idx)) | |
1699 | t = m_data[m_data_cnt]; | |
1700 | else if (TYPE_PRECISION (rhs_type) > limb_prec | |
1701 | && integer_onep (idx)) | |
1702 | t = m_data[m_data_cnt + 1]; | |
1703 | else | |
1704 | t = m_data[m_data_cnt + 2]; | |
1705 | if (!useless_type_conversion_p (type, m_limb_type)) | |
1706 | t = add_cast (type, t); | |
1707 | m_data_cnt += 3; | |
1708 | return t; | |
1709 | } | |
1710 | g = gimple_build_cond (NE_EXPR, idx, size_zero_node, | |
1711 | NULL_TREE, NULL_TREE); | |
1712 | edge e2, e3, e4 = NULL; | |
1713 | if_then (g, profile_probability::likely (), e2, e3); | |
1714 | if (m_data[m_data_cnt + 1]) | |
1715 | { | |
1716 | g = gimple_build_cond (EQ_EXPR, idx, size_one_node, | |
1717 | NULL_TREE, NULL_TREE); | |
1718 | insert_before (g); | |
1719 | edge e5 = split_block (gsi_bb (m_gsi), g); | |
1720 | e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE); | |
1721 | e2 = find_edge (e5->dest, e2->dest); | |
1722 | e4->probability = profile_probability::unlikely (); | |
1723 | e5->flags = EDGE_FALSE_VALUE; | |
1724 | e5->probability = e4->probability.invert (); | |
1725 | } | |
1726 | m_gsi = gsi_after_labels (e2->dest); | |
1727 | t = make_ssa_name (m_limb_type); | |
1728 | gphi *phi = create_phi_node (t, e2->dest); | |
1729 | add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION); | |
1730 | add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION); | |
1731 | if (e4) | |
1732 | add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION); | |
1733 | m_data_cnt += 3; | |
1734 | return t; | |
1735 | } | |
1736 | return NULL_TREE; | |
1737 | } | |
1738 | ||
9d2bc5de JJ |
1739 | /* Helper function for handle_stmt method, handle a BIT_FIELD_REF. */ |
1740 | ||
1741 | tree | |
1742 | bitint_large_huge::handle_bit_field_ref (tree op, tree idx) | |
1743 | { | |
1744 | if (tree_fits_uhwi_p (idx)) | |
1745 | { | |
1746 | if (m_first) | |
1747 | m_data.safe_push (NULL); | |
1748 | ++m_data_cnt; | |
1749 | unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type)); | |
1750 | tree bfr = build3 (BIT_FIELD_REF, m_limb_type, | |
1751 | TREE_OPERAND (op, 0), | |
1752 | TYPE_SIZE (m_limb_type), | |
1753 | size_binop (PLUS_EXPR, TREE_OPERAND (op, 2), | |
1754 | bitsize_int (tree_to_uhwi (idx) * sz))); | |
1755 | tree r = make_ssa_name (m_limb_type); | |
1756 | gimple *g = gimple_build_assign (r, bfr); | |
1757 | insert_before (g); | |
1758 | tree type = limb_access_type (TREE_TYPE (op), idx); | |
1759 | if (!useless_type_conversion_p (type, m_limb_type)) | |
1760 | r = add_cast (type, r); | |
1761 | return r; | |
1762 | } | |
1763 | tree var; | |
1764 | if (m_first) | |
1765 | { | |
1766 | unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))); | |
1767 | machine_mode mode; | |
1768 | tree type, bfr; | |
1769 | if (bitwise_mode_for_size (sz).exists (&mode) | |
1770 | && known_eq (GET_MODE_BITSIZE (mode), sz)) | |
1771 | type = bitwise_type_for_mode (mode); | |
1772 | else | |
1773 | { | |
1774 | mode = VOIDmode; | |
1775 | type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0))); | |
1776 | } | |
1777 | if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op))) | |
1778 | type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op))); | |
1779 | var = create_tmp_var (type); | |
1780 | TREE_ADDRESSABLE (var) = 1; | |
1781 | gimple *g; | |
1782 | if (mode != VOIDmode) | |
1783 | { | |
1784 | bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0), | |
1785 | TYPE_SIZE (type), TREE_OPERAND (op, 2)); | |
1786 | g = gimple_build_assign (make_ssa_name (type), | |
1787 | BIT_FIELD_REF, bfr); | |
1788 | gimple_set_location (g, m_loc); | |
1789 | gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT); | |
1790 | bfr = gimple_assign_lhs (g); | |
1791 | } | |
1792 | else | |
1793 | bfr = TREE_OPERAND (op, 0); | |
1794 | g = gimple_build_assign (var, bfr); | |
1795 | gimple_set_location (g, m_loc); | |
1796 | gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT); | |
1797 | if (mode == VOIDmode) | |
1798 | { | |
1799 | unsigned HOST_WIDE_INT nelts | |
1800 | = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec); | |
1801 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
1802 | var = build2 (MEM_REF, atype, build_fold_addr_expr (var), | |
1803 | build_int_cst (build_pointer_type (type), | |
1804 | tree_to_uhwi (TREE_OPERAND (op, 2)) | |
1805 | / BITS_PER_UNIT)); | |
1806 | } | |
1807 | m_data.safe_push (var); | |
1808 | } | |
1809 | else | |
1810 | var = unshare_expr (m_data[m_data_cnt]); | |
1811 | ++m_data_cnt; | |
1812 | var = limb_access (TREE_TYPE (op), var, idx, false); | |
1813 | tree r = make_ssa_name (m_limb_type); | |
1814 | gimple *g = gimple_build_assign (r, var); | |
1815 | insert_before (g); | |
1816 | return r; | |
1817 | } | |
1818 | ||
6c124873 JJ |
1819 | /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE |
1820 | is an older EH edge, and except for virtual PHIs duplicate the | |
1821 | PHI argument from the EH_EDGE to the new EH edge. */ | |
1822 | ||
1823 | static void | |
1824 | add_eh_edge (basic_block src, edge eh_edge) | |
1825 | { | |
1826 | edge e = make_edge (src, eh_edge->dest, EDGE_EH); | |
1827 | e->probability = profile_probability::very_unlikely (); | |
1828 | for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest); | |
1829 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
1830 | { | |
1831 | gphi *phi = gsi.phi (); | |
1832 | tree lhs = gimple_phi_result (phi); | |
1833 | if (virtual_operand_p (lhs)) | |
1834 | continue; | |
1835 | const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx); | |
1836 | add_phi_arg (phi, arg->def, e, arg->locus); | |
1837 | } | |
1838 | } | |
1839 | ||
a9d6c7fb JJ |
1840 | /* Helper function for handle_stmt method, handle a load from memory. */ |
1841 | ||
1842 | tree | |
1843 | bitint_large_huge::handle_load (gimple *stmt, tree idx) | |
1844 | { | |
1845 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
1846 | tree rhs_type = TREE_TYPE (rhs1); | |
1847 | bool eh = stmt_ends_bb_p (stmt); | |
1848 | edge eh_edge = NULL; | |
1849 | gimple *g; | |
1850 | ||
1851 | if (eh) | |
1852 | { | |
1853 | edge_iterator ei; | |
1854 | basic_block bb = gimple_bb (stmt); | |
1855 | ||
1856 | FOR_EACH_EDGE (eh_edge, ei, bb->succs) | |
1857 | if (eh_edge->flags & EDGE_EH) | |
1858 | break; | |
1859 | } | |
1860 | ||
1861 | if (TREE_CODE (rhs1) == COMPONENT_REF | |
1862 | && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1))) | |
1863 | { | |
1864 | tree fld = TREE_OPERAND (rhs1, 1); | |
1865 | /* For little-endian, we can allow as inputs bit-fields | |
1866 | which start at a limb boundary. */ | |
1867 | gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))); | |
1868 | if (DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1)) | |
1869 | && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0) | |
1870 | goto normal_load; | |
1871 | /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of UNITS_PER_BIT, | |
1872 | handle it normally for now. */ | |
1873 | if ((tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0) | |
1874 | goto normal_load; | |
1875 | tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld); | |
1876 | poly_int64 bitoffset; | |
1877 | poly_uint64 field_offset, repr_offset; | |
1878 | bool var_field_off = false; | |
1879 | if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset) | |
1880 | && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset)) | |
1881 | bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT; | |
1882 | else | |
1883 | { | |
1884 | bitoffset = 0; | |
1885 | var_field_off = true; | |
1886 | } | |
1887 | bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) | |
1888 | - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr))); | |
1889 | tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr), | |
1890 | TREE_OPERAND (rhs1, 0), repr, | |
1891 | var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE); | |
1892 | HOST_WIDE_INT bo = bitoffset.to_constant (); | |
1893 | unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec; | |
1894 | unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec; | |
1895 | if (m_first) | |
1896 | { | |
1897 | if (m_upwards) | |
1898 | { | |
1899 | gimple_stmt_iterator save_gsi = m_gsi; | |
1900 | m_gsi = m_init_gsi; | |
1901 | if (gsi_end_p (m_gsi)) | |
1902 | m_gsi = gsi_after_labels (gsi_bb (m_gsi)); | |
1903 | else | |
1904 | gsi_next (&m_gsi); | |
0613b12d | 1905 | tree t = limb_access (NULL_TREE, nrhs1, size_int (bo_idx), true); |
a9d6c7fb JJ |
1906 | tree iv = make_ssa_name (m_limb_type); |
1907 | g = gimple_build_assign (iv, t); | |
1908 | insert_before (g); | |
1909 | if (eh) | |
1910 | { | |
1911 | maybe_duplicate_eh_stmt (g, stmt); | |
1912 | if (eh_edge) | |
1913 | { | |
1914 | edge e = split_block (gsi_bb (m_gsi), g); | |
6c124873 | 1915 | add_eh_edge (e->src, eh_edge); |
9a96a9e4 JJ |
1916 | m_gsi = gsi_after_labels (e->dest); |
1917 | if (gsi_bb (save_gsi) == e->src) | |
1918 | { | |
1919 | if (gsi_end_p (save_gsi)) | |
1920 | save_gsi = gsi_end_bb (e->dest); | |
1921 | else | |
1922 | save_gsi = gsi_for_stmt (gsi_stmt (save_gsi)); | |
1923 | } | |
1924 | if (m_preheader_bb == e->src) | |
1925 | m_preheader_bb = e->dest; | |
a9d6c7fb JJ |
1926 | } |
1927 | } | |
9a96a9e4 JJ |
1928 | m_init_gsi = m_gsi; |
1929 | if (gsi_end_p (m_init_gsi)) | |
1930 | m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi)); | |
1931 | else | |
1932 | gsi_prev (&m_init_gsi); | |
a9d6c7fb JJ |
1933 | m_gsi = save_gsi; |
1934 | tree out; | |
1935 | prepare_data_in_out (iv, idx, &out); | |
1936 | out = m_data[m_data_cnt]; | |
1937 | m_data.safe_push (out); | |
1938 | } | |
1939 | else | |
1940 | { | |
1941 | m_data.safe_push (NULL_TREE); | |
1942 | m_data.safe_push (NULL_TREE); | |
1943 | m_data.safe_push (NULL_TREE); | |
1944 | } | |
1945 | } | |
1946 | ||
1947 | tree nidx0 = NULL_TREE, nidx1; | |
1948 | tree iv = m_data[m_data_cnt]; | |
1949 | if (m_cast_conditional && iv) | |
1950 | { | |
1951 | gcc_assert (!m_bitfld_load); | |
1952 | m_bitfld_load = m_data_cnt; | |
1953 | } | |
1954 | if (tree_fits_uhwi_p (idx)) | |
1955 | { | |
1956 | unsigned prec = TYPE_PRECISION (rhs_type); | |
1957 | unsigned HOST_WIDE_INT i = tree_to_uhwi (idx); | |
1958 | gcc_assert (i * limb_prec < prec); | |
1959 | nidx1 = size_int (i + bo_idx + 1); | |
1960 | if ((i + 1) * limb_prec > prec) | |
1961 | { | |
1962 | prec %= limb_prec; | |
1963 | if (prec + bo_bit <= (unsigned) limb_prec) | |
1964 | nidx1 = NULL_TREE; | |
1965 | } | |
1966 | if (!iv) | |
1967 | nidx0 = size_int (i + bo_idx); | |
1968 | } | |
1969 | else | |
1970 | { | |
1971 | if (!iv) | |
1972 | { | |
1973 | if (bo_idx == 0) | |
1974 | nidx0 = idx; | |
1975 | else | |
1976 | { | |
1977 | nidx0 = make_ssa_name (sizetype); | |
1978 | g = gimple_build_assign (nidx0, PLUS_EXPR, idx, | |
1979 | size_int (bo_idx)); | |
1980 | insert_before (g); | |
1981 | } | |
1982 | } | |
1983 | nidx1 = make_ssa_name (sizetype); | |
1984 | g = gimple_build_assign (nidx1, PLUS_EXPR, idx, | |
1985 | size_int (bo_idx + 1)); | |
1986 | insert_before (g); | |
1987 | } | |
1988 | ||
1989 | tree iv2 = NULL_TREE; | |
1990 | if (nidx0) | |
1991 | { | |
0613b12d | 1992 | tree t = limb_access (NULL_TREE, nrhs1, nidx0, true); |
a9d6c7fb JJ |
1993 | iv = make_ssa_name (m_limb_type); |
1994 | g = gimple_build_assign (iv, t); | |
1995 | insert_before (g); | |
1996 | gcc_assert (!eh); | |
1997 | } | |
1998 | if (nidx1) | |
1999 | { | |
2000 | bool conditional = m_var_msb && !tree_fits_uhwi_p (idx); | |
2001 | unsigned prec = TYPE_PRECISION (rhs_type); | |
2002 | if (conditional) | |
2003 | { | |
2004 | if ((prec % limb_prec) == 0 | |
2005 | || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec)) | |
2006 | conditional = false; | |
2007 | } | |
2008 | edge edge_true = NULL, edge_false = NULL; | |
2009 | if (conditional) | |
2010 | { | |
2011 | g = gimple_build_cond (NE_EXPR, idx, | |
2012 | size_int (prec / limb_prec), | |
2013 | NULL_TREE, NULL_TREE); | |
2014 | if_then (g, profile_probability::likely (), | |
2015 | edge_true, edge_false); | |
2016 | } | |
0613b12d | 2017 | tree t = limb_access (NULL_TREE, nrhs1, nidx1, true); |
a9d6c7fb JJ |
2018 | if (m_upwards_2limb |
2019 | && !m_first | |
2020 | && !m_bitfld_load | |
2021 | && !tree_fits_uhwi_p (idx)) | |
2022 | iv2 = m_data[m_data_cnt + 1]; | |
2023 | else | |
2024 | iv2 = make_ssa_name (m_limb_type); | |
2025 | g = gimple_build_assign (iv2, t); | |
2026 | insert_before (g); | |
2027 | if (eh) | |
2028 | { | |
2029 | maybe_duplicate_eh_stmt (g, stmt); | |
2030 | if (eh_edge) | |
2031 | { | |
2032 | edge e = split_block (gsi_bb (m_gsi), g); | |
2033 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 | 2034 | add_eh_edge (e->src, eh_edge); |
a9d6c7fb JJ |
2035 | } |
2036 | } | |
2037 | if (conditional) | |
2038 | { | |
2039 | tree iv3 = make_ssa_name (m_limb_type); | |
2040 | if (eh) | |
2041 | edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest); | |
2042 | gphi *phi = create_phi_node (iv3, edge_true->dest); | |
2043 | add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION); | |
2044 | add_phi_arg (phi, build_zero_cst (m_limb_type), | |
2045 | edge_false, UNKNOWN_LOCATION); | |
2046 | m_gsi = gsi_after_labels (edge_true->dest); | |
456e10f2 | 2047 | iv2 = iv3; |
a9d6c7fb JJ |
2048 | } |
2049 | } | |
2050 | g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR, | |
2051 | iv, build_int_cst (unsigned_type_node, bo_bit)); | |
2052 | insert_before (g); | |
2053 | iv = gimple_assign_lhs (g); | |
2054 | if (iv2) | |
2055 | { | |
2056 | g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR, | |
2057 | iv2, build_int_cst (unsigned_type_node, | |
2058 | limb_prec - bo_bit)); | |
2059 | insert_before (g); | |
2060 | g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR, | |
2061 | gimple_assign_lhs (g), iv); | |
2062 | insert_before (g); | |
2063 | iv = gimple_assign_lhs (g); | |
2064 | if (m_data[m_data_cnt]) | |
2065 | m_data[m_data_cnt] = iv2; | |
2066 | } | |
2067 | if (tree_fits_uhwi_p (idx)) | |
2068 | { | |
2069 | tree atype = limb_access_type (rhs_type, idx); | |
2070 | if (!useless_type_conversion_p (atype, TREE_TYPE (iv))) | |
2071 | iv = add_cast (atype, iv); | |
2072 | } | |
2073 | m_data_cnt += 3; | |
2074 | return iv; | |
2075 | } | |
2076 | ||
2077 | normal_load: | |
2078 | /* Use write_p = true for loads with EH edges to make | |
2079 | sure limb_access doesn't add a cast as separate | |
2080 | statement after it. */ | |
2081 | rhs1 = limb_access (rhs_type, rhs1, idx, eh); | |
2082 | tree ret = make_ssa_name (TREE_TYPE (rhs1)); | |
2083 | g = gimple_build_assign (ret, rhs1); | |
2084 | insert_before (g); | |
2085 | if (eh) | |
2086 | { | |
2087 | maybe_duplicate_eh_stmt (g, stmt); | |
2088 | if (eh_edge) | |
2089 | { | |
2090 | edge e = split_block (gsi_bb (m_gsi), g); | |
2091 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 | 2092 | add_eh_edge (e->src, eh_edge); |
a9d6c7fb JJ |
2093 | } |
2094 | if (tree_fits_uhwi_p (idx)) | |
2095 | { | |
2096 | tree atype = limb_access_type (rhs_type, idx); | |
2097 | if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1))) | |
2098 | ret = add_cast (atype, ret); | |
2099 | } | |
2100 | } | |
2101 | return ret; | |
2102 | } | |
2103 | ||
2104 | /* Return a limb IDX from a mergeable statement STMT. */ | |
2105 | ||
2106 | tree | |
2107 | bitint_large_huge::handle_stmt (gimple *stmt, tree idx) | |
2108 | { | |
2109 | tree lhs, rhs1, rhs2 = NULL_TREE; | |
2110 | gimple *g; | |
2111 | switch (gimple_code (stmt)) | |
2112 | { | |
2113 | case GIMPLE_ASSIGN: | |
2114 | if (gimple_assign_load_p (stmt)) | |
2115 | return handle_load (stmt, idx); | |
2116 | switch (gimple_assign_rhs_code (stmt)) | |
2117 | { | |
2118 | case BIT_AND_EXPR: | |
2119 | case BIT_IOR_EXPR: | |
2120 | case BIT_XOR_EXPR: | |
2121 | rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx); | |
2122 | /* FALLTHRU */ | |
2123 | case BIT_NOT_EXPR: | |
2124 | rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx); | |
2125 | lhs = make_ssa_name (TREE_TYPE (rhs1)); | |
2126 | g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt), | |
2127 | rhs1, rhs2); | |
2128 | insert_before (g); | |
2129 | return lhs; | |
2130 | case PLUS_EXPR: | |
2131 | case MINUS_EXPR: | |
2132 | rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx); | |
2133 | rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx); | |
2134 | return handle_plus_minus (gimple_assign_rhs_code (stmt), | |
2135 | rhs1, rhs2, idx); | |
2136 | case NEGATE_EXPR: | |
2137 | rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx); | |
2138 | rhs1 = build_zero_cst (TREE_TYPE (rhs2)); | |
2139 | return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx); | |
2140 | case LSHIFT_EXPR: | |
2141 | return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt), | |
2142 | idx), | |
2143 | gimple_assign_rhs2 (stmt), idx); | |
2144 | case SSA_NAME: | |
2145 | case INTEGER_CST: | |
2146 | return handle_operand (gimple_assign_rhs1 (stmt), idx); | |
2147 | CASE_CONVERT: | |
a9d6c7fb JJ |
2148 | return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)), |
2149 | gimple_assign_rhs1 (stmt), idx); | |
2fb78d43 JJ |
2150 | case VIEW_CONVERT_EXPR: |
2151 | return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)), | |
2152 | TREE_OPERAND (gimple_assign_rhs1 (stmt), 0), | |
2153 | idx); | |
9d2bc5de JJ |
2154 | case BIT_FIELD_REF: |
2155 | return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx); | |
a9d6c7fb JJ |
2156 | default: |
2157 | break; | |
2158 | } | |
2159 | break; | |
2160 | default: | |
2161 | break; | |
2162 | } | |
2163 | gcc_unreachable (); | |
2164 | } | |
2165 | ||
2166 | /* Return minimum precision of OP at STMT. | |
2167 | Positive value is minimum precision above which all bits | |
2168 | are zero, negative means all bits above negation of the | |
2169 | value are copies of the sign bit. */ | |
2170 | ||
2171 | static int | |
2172 | range_to_prec (tree op, gimple *stmt) | |
2173 | { | |
2174 | int_range_max r; | |
2175 | wide_int w; | |
2176 | tree type = TREE_TYPE (op); | |
2177 | unsigned int prec = TYPE_PRECISION (type); | |
2178 | ||
2179 | if (!optimize | |
09b51246 JJ |
2180 | || !get_range_query (cfun)->range_of_expr (r, op, stmt) |
2181 | || r.undefined_p ()) | |
a9d6c7fb JJ |
2182 | { |
2183 | if (TYPE_UNSIGNED (type)) | |
2184 | return prec; | |
2185 | else | |
e3be66df | 2186 | return MIN ((int) -prec, -2); |
a9d6c7fb JJ |
2187 | } |
2188 | ||
2189 | if (!TYPE_UNSIGNED (TREE_TYPE (op))) | |
2190 | { | |
2191 | w = r.lower_bound (); | |
2192 | if (wi::neg_p (w)) | |
2193 | { | |
2194 | int min_prec1 = wi::min_precision (w, SIGNED); | |
2195 | w = r.upper_bound (); | |
2196 | int min_prec2 = wi::min_precision (w, SIGNED); | |
2197 | int min_prec = MAX (min_prec1, min_prec2); | |
2198 | return MIN (-min_prec, -2); | |
2199 | } | |
2200 | } | |
2201 | ||
2202 | w = r.upper_bound (); | |
2203 | int min_prec = wi::min_precision (w, UNSIGNED); | |
2204 | return MAX (min_prec, 1); | |
2205 | } | |
2206 | ||
2207 | /* Return address of the first limb of OP and write into *PREC | |
2208 | its precision. If positive, the operand is zero extended | |
2209 | from that precision, if it is negative, the operand is sign-extended | |
2210 | from -*PREC. If PREC_STORED is NULL, it is the toplevel call, | |
2211 | otherwise *PREC_STORED is prec from the innermost call without | |
2212 | range optimizations. */ | |
2213 | ||
2214 | tree | |
2215 | bitint_large_huge::handle_operand_addr (tree op, gimple *stmt, | |
2216 | int *prec_stored, int *prec) | |
2217 | { | |
2218 | wide_int w; | |
2219 | location_t loc_save = m_loc; | |
2220 | if ((TREE_CODE (TREE_TYPE (op)) != BITINT_TYPE | |
2221 | || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large) | |
2222 | && TREE_CODE (op) != INTEGER_CST) | |
2223 | { | |
2224 | do_int: | |
2225 | *prec = range_to_prec (op, stmt); | |
2226 | bitint_prec_kind kind = bitint_prec_small; | |
2227 | gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op))); | |
2228 | if (TREE_CODE (TREE_TYPE (op)) == BITINT_TYPE) | |
2229 | kind = bitint_precision_kind (TREE_TYPE (op)); | |
2230 | if (kind == bitint_prec_middle) | |
2231 | { | |
2232 | tree type = NULL_TREE; | |
2233 | op = maybe_cast_middle_bitint (&m_gsi, op, type); | |
2234 | } | |
2235 | tree op_type = TREE_TYPE (op); | |
2236 | unsigned HOST_WIDE_INT nelts | |
2237 | = CEIL (TYPE_PRECISION (op_type), limb_prec); | |
2238 | /* Add support for 3 or more limbs filled in from normal | |
2239 | integral type if this assert fails. If no target chooses | |
2240 | limb mode smaller than half of largest supported normal | |
2241 | integral type, this will not be needed. */ | |
2242 | gcc_assert (nelts <= 2); | |
2243 | if (prec_stored) | |
2244 | *prec_stored = (TYPE_UNSIGNED (op_type) | |
2245 | ? TYPE_PRECISION (op_type) | |
2246 | : -TYPE_PRECISION (op_type)); | |
2247 | if (*prec <= limb_prec && *prec >= -limb_prec) | |
2248 | { | |
2249 | nelts = 1; | |
2250 | if (prec_stored) | |
2251 | { | |
2252 | if (TYPE_UNSIGNED (op_type)) | |
2253 | { | |
2254 | if (*prec_stored > limb_prec) | |
2255 | *prec_stored = limb_prec; | |
2256 | } | |
2257 | else if (*prec_stored < -limb_prec) | |
2258 | *prec_stored = -limb_prec; | |
2259 | } | |
2260 | } | |
2261 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
2262 | tree var = create_tmp_var (atype); | |
2263 | tree t1 = op; | |
2264 | if (!useless_type_conversion_p (m_limb_type, op_type)) | |
2265 | t1 = add_cast (m_limb_type, t1); | |
2266 | tree v = build4 (ARRAY_REF, m_limb_type, var, size_zero_node, | |
2267 | NULL_TREE, NULL_TREE); | |
2268 | gimple *g = gimple_build_assign (v, t1); | |
2269 | insert_before (g); | |
2270 | if (nelts > 1) | |
2271 | { | |
2272 | tree lp = build_int_cst (unsigned_type_node, limb_prec); | |
2273 | g = gimple_build_assign (make_ssa_name (op_type), | |
2274 | RSHIFT_EXPR, op, lp); | |
2275 | insert_before (g); | |
2276 | tree t2 = gimple_assign_lhs (g); | |
2277 | t2 = add_cast (m_limb_type, t2); | |
2278 | v = build4 (ARRAY_REF, m_limb_type, var, size_one_node, | |
2279 | NULL_TREE, NULL_TREE); | |
2280 | g = gimple_build_assign (v, t2); | |
2281 | insert_before (g); | |
2282 | } | |
2283 | tree ret = build_fold_addr_expr (var); | |
2284 | if (!stmt_ends_bb_p (gsi_stmt (m_gsi))) | |
2285 | { | |
de072b52 | 2286 | tree clobber = build_clobber (atype, CLOBBER_STORAGE_END); |
a9d6c7fb JJ |
2287 | g = gimple_build_assign (var, clobber); |
2288 | gsi_insert_after (&m_gsi, g, GSI_SAME_STMT); | |
2289 | } | |
2290 | m_loc = loc_save; | |
2291 | return ret; | |
2292 | } | |
2293 | switch (TREE_CODE (op)) | |
2294 | { | |
2295 | case SSA_NAME: | |
2296 | if (m_names == NULL | |
2297 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op))) | |
2298 | { | |
2299 | gimple *g = SSA_NAME_DEF_STMT (op); | |
2300 | tree ret; | |
2301 | m_loc = gimple_location (g); | |
2302 | if (gimple_assign_load_p (g)) | |
2303 | { | |
2304 | *prec = range_to_prec (op, NULL); | |
2305 | if (prec_stored) | |
2306 | *prec_stored = (TYPE_UNSIGNED (TREE_TYPE (op)) | |
2307 | ? TYPE_PRECISION (TREE_TYPE (op)) | |
2308 | : -TYPE_PRECISION (TREE_TYPE (op))); | |
2309 | ret = build_fold_addr_expr (gimple_assign_rhs1 (g)); | |
2310 | ret = force_gimple_operand_gsi (&m_gsi, ret, true, | |
2311 | NULL_TREE, true, GSI_SAME_STMT); | |
2312 | } | |
2313 | else if (gimple_code (g) == GIMPLE_NOP) | |
2314 | { | |
09b51246 JJ |
2315 | *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec; |
2316 | if (prec_stored) | |
2317 | *prec_stored = *prec; | |
a9d6c7fb JJ |
2318 | tree var = create_tmp_var (m_limb_type); |
2319 | TREE_ADDRESSABLE (var) = 1; | |
2320 | ret = build_fold_addr_expr (var); | |
2321 | if (!stmt_ends_bb_p (gsi_stmt (m_gsi))) | |
2322 | { | |
de072b52 JM |
2323 | tree clobber = build_clobber (m_limb_type, |
2324 | CLOBBER_STORAGE_END); | |
a9d6c7fb JJ |
2325 | g = gimple_build_assign (var, clobber); |
2326 | gsi_insert_after (&m_gsi, g, GSI_SAME_STMT); | |
2327 | } | |
2328 | } | |
2329 | else | |
2330 | { | |
2331 | gcc_assert (gimple_assign_cast_p (g)); | |
2332 | tree rhs1 = gimple_assign_rhs1 (g); | |
2333 | bitint_prec_kind kind = bitint_prec_small; | |
90ac839a JJ |
2334 | if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR) |
2335 | rhs1 = TREE_OPERAND (rhs1, 0); | |
a9d6c7fb JJ |
2336 | gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))); |
2337 | if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE) | |
2338 | kind = bitint_precision_kind (TREE_TYPE (rhs1)); | |
2339 | if (kind >= bitint_prec_large) | |
2340 | { | |
2341 | tree lhs_type = TREE_TYPE (op); | |
2342 | tree rhs_type = TREE_TYPE (rhs1); | |
2343 | int prec_stored_val = 0; | |
2344 | ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec); | |
2345 | if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type)) | |
2346 | { | |
2347 | if (TYPE_UNSIGNED (lhs_type) | |
2348 | && !TYPE_UNSIGNED (rhs_type)) | |
2349 | gcc_assert (*prec >= 0 || prec_stored == NULL); | |
2350 | } | |
2351 | else | |
2352 | { | |
2353 | if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type)) | |
2354 | ; | |
2355 | else if (TYPE_UNSIGNED (lhs_type)) | |
2356 | { | |
2357 | gcc_assert (*prec > 0 | |
2358 | || prec_stored_val > 0 | |
2359 | || (-prec_stored_val | |
2360 | >= TYPE_PRECISION (lhs_type))); | |
2361 | *prec = TYPE_PRECISION (lhs_type); | |
2362 | } | |
2363 | else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type)) | |
2364 | ; | |
2365 | else | |
2366 | *prec = -TYPE_PRECISION (lhs_type); | |
2367 | } | |
2368 | } | |
2369 | else | |
2370 | { | |
2371 | op = rhs1; | |
2372 | stmt = g; | |
2373 | goto do_int; | |
2374 | } | |
2375 | } | |
2376 | m_loc = loc_save; | |
2377 | return ret; | |
2378 | } | |
2379 | else | |
2380 | { | |
2381 | int p = var_to_partition (m_map, op); | |
2382 | gcc_assert (m_vars[p] != NULL_TREE); | |
2383 | *prec = range_to_prec (op, stmt); | |
2384 | if (prec_stored) | |
2385 | *prec_stored = (TYPE_UNSIGNED (TREE_TYPE (op)) | |
2386 | ? TYPE_PRECISION (TREE_TYPE (op)) | |
2387 | : -TYPE_PRECISION (TREE_TYPE (op))); | |
2388 | return build_fold_addr_expr (m_vars[p]); | |
2389 | } | |
2390 | case INTEGER_CST: | |
2391 | unsigned int min_prec, mp; | |
2392 | tree type; | |
2393 | w = wi::to_wide (op); | |
2394 | if (tree_int_cst_sgn (op) >= 0) | |
2395 | { | |
2396 | min_prec = wi::min_precision (w, UNSIGNED); | |
2397 | *prec = MAX (min_prec, 1); | |
2398 | } | |
2399 | else | |
2400 | { | |
2401 | min_prec = wi::min_precision (w, SIGNED); | |
2402 | *prec = MIN ((int) -min_prec, -2); | |
2403 | } | |
2404 | mp = CEIL (min_prec, limb_prec) * limb_prec; | |
9bfebcb1 JJ |
2405 | if (mp == 0) |
2406 | mp = 1; | |
f77a8798 JJ |
2407 | if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op)) |
2408 | && (TREE_CODE (TREE_TYPE (op)) == BITINT_TYPE | |
2409 | || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec)) | |
a9d6c7fb JJ |
2410 | type = TREE_TYPE (op); |
2411 | else | |
2412 | type = build_bitint_type (mp, 1); | |
2413 | if (TREE_CODE (type) != BITINT_TYPE | |
2414 | || bitint_precision_kind (type) == bitint_prec_small) | |
2415 | { | |
2416 | if (TYPE_PRECISION (type) <= limb_prec) | |
2417 | type = m_limb_type; | |
2418 | else | |
f77a8798 JJ |
2419 | { |
2420 | while (bitint_precision_kind (mp) == bitint_prec_small) | |
2421 | mp += limb_prec; | |
2422 | /* This case is for targets which e.g. have 64-bit | |
2423 | limb but categorize up to 128-bits _BitInts as | |
2424 | small. We could use type of m_limb_type[2] and | |
2425 | similar instead to save space. */ | |
2426 | type = build_bitint_type (mp, 1); | |
2427 | } | |
a9d6c7fb JJ |
2428 | } |
2429 | if (prec_stored) | |
2430 | { | |
2431 | if (tree_int_cst_sgn (op) >= 0) | |
2432 | *prec_stored = MAX (TYPE_PRECISION (type), 1); | |
2433 | else | |
2434 | *prec_stored = MIN ((int) -TYPE_PRECISION (type), -2); | |
2435 | } | |
2436 | op = tree_output_constant_def (fold_convert (type, op)); | |
2437 | return build_fold_addr_expr (op); | |
2438 | default: | |
2439 | gcc_unreachable (); | |
2440 | } | |
2441 | } | |
2442 | ||
2443 | /* Helper function, create a loop before the current location, | |
2444 | start with sizetype INIT value from the preheader edge. Return | |
2445 | a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses | |
2446 | from the latch edge. */ | |
2447 | ||
2448 | tree | |
2449 | bitint_large_huge::create_loop (tree init, tree *idx_next) | |
2450 | { | |
2451 | if (!gsi_end_p (m_gsi)) | |
2452 | gsi_prev (&m_gsi); | |
2453 | else | |
2454 | m_gsi = gsi_last_bb (gsi_bb (m_gsi)); | |
2455 | edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi)); | |
2456 | edge e2 = split_block (e1->dest, (gimple *) NULL); | |
2457 | edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE); | |
2458 | e3->probability = profile_probability::very_unlikely (); | |
2459 | e2->flags = EDGE_FALSE_VALUE; | |
2460 | e2->probability = e3->probability.invert (); | |
2461 | tree idx = make_ssa_name (sizetype); | |
2462 | gphi *phi = create_phi_node (idx, e1->dest); | |
2463 | add_phi_arg (phi, init, e1, UNKNOWN_LOCATION); | |
2464 | *idx_next = make_ssa_name (sizetype); | |
2465 | add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION); | |
2466 | m_gsi = gsi_after_labels (e1->dest); | |
2467 | m_bb = e1->dest; | |
2468 | m_preheader_bb = e1->src; | |
2469 | class loop *loop = alloc_loop (); | |
2470 | loop->header = e1->dest; | |
2471 | add_loop (loop, e1->src->loop_father); | |
2472 | return idx; | |
2473 | } | |
2474 | ||
2475 | /* Lower large/huge _BitInt statement mergeable or similar STMT which can be | |
2476 | lowered using iteration from the least significant limb up to the most | |
2477 | significant limb. For large _BitInt it is emitted as straight line code | |
2478 | before current location, for huge _BitInt as a loop handling two limbs | |
2479 | at once, followed by handling up to limbs in straight line code (at most | |
2480 | one full and one partial limb). It can also handle EQ_EXPR/NE_EXPR | |
2481 | comparisons, in that case CMP_CODE should be the comparison code and | |
2482 | CMP_OP1/CMP_OP2 the comparison operands. */ | |
2483 | ||
2484 | tree | |
2485 | bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code, | |
2486 | tree cmp_op1, tree cmp_op2) | |
2487 | { | |
2488 | bool eq_p = cmp_code != ERROR_MARK; | |
2489 | tree type; | |
2490 | if (eq_p) | |
2491 | type = TREE_TYPE (cmp_op1); | |
2492 | else | |
2493 | type = TREE_TYPE (gimple_assign_lhs (stmt)); | |
2494 | gcc_assert (TREE_CODE (type) == BITINT_TYPE); | |
2495 | bitint_prec_kind kind = bitint_precision_kind (type); | |
2496 | gcc_assert (kind >= bitint_prec_large); | |
2497 | gimple *g; | |
2498 | tree lhs = gimple_get_lhs (stmt); | |
2499 | tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE; | |
2500 | if (lhs | |
2501 | && TREE_CODE (lhs) == SSA_NAME | |
2502 | && TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE | |
2503 | && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large) | |
2504 | { | |
2505 | int p = var_to_partition (m_map, lhs); | |
2506 | gcc_assert (m_vars[p] != NULL_TREE); | |
2507 | m_lhs = lhs = m_vars[p]; | |
2508 | } | |
2509 | unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type); | |
2510 | bool sext = false; | |
2511 | tree ext = NULL_TREE, store_operand = NULL_TREE; | |
2512 | bool eh = false; | |
2513 | basic_block eh_pad = NULL; | |
2514 | tree nlhs = NULL_TREE; | |
2515 | unsigned HOST_WIDE_INT bo_idx = 0; | |
2516 | unsigned HOST_WIDE_INT bo_bit = 0; | |
2517 | tree bf_cur = NULL_TREE, bf_next = NULL_TREE; | |
2518 | if (gimple_store_p (stmt)) | |
2519 | { | |
2520 | store_operand = gimple_assign_rhs1 (stmt); | |
2521 | eh = stmt_ends_bb_p (stmt); | |
2522 | if (eh) | |
2523 | { | |
2524 | edge e; | |
2525 | edge_iterator ei; | |
2526 | basic_block bb = gimple_bb (stmt); | |
2527 | ||
2528 | FOR_EACH_EDGE (e, ei, bb->succs) | |
2529 | if (e->flags & EDGE_EH) | |
2530 | { | |
2531 | eh_pad = e->dest; | |
2532 | break; | |
2533 | } | |
2534 | } | |
2535 | if (TREE_CODE (lhs) == COMPONENT_REF | |
2536 | && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1))) | |
2537 | { | |
2538 | tree fld = TREE_OPERAND (lhs, 1); | |
2539 | gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))); | |
2540 | tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld); | |
2541 | poly_int64 bitoffset; | |
2542 | poly_uint64 field_offset, repr_offset; | |
2543 | if ((tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0) | |
2544 | nlhs = lhs; | |
2545 | else | |
2546 | { | |
2547 | bool var_field_off = false; | |
2548 | if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset) | |
2549 | && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset)) | |
2550 | bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT; | |
2551 | else | |
2552 | { | |
2553 | bitoffset = 0; | |
2554 | var_field_off = true; | |
2555 | } | |
2556 | bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) | |
2557 | - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr))); | |
2558 | nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr), | |
2559 | TREE_OPERAND (lhs, 0), repr, | |
2560 | var_field_off | |
2561 | ? TREE_OPERAND (lhs, 2) : NULL_TREE); | |
2562 | HOST_WIDE_INT bo = bitoffset.to_constant (); | |
2563 | bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec; | |
2564 | bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec; | |
2565 | } | |
2566 | } | |
2567 | } | |
2568 | if ((store_operand | |
2569 | && TREE_CODE (store_operand) == SSA_NAME | |
2570 | && (m_names == NULL | |
2571 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand))) | |
2572 | && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand))) | |
2573 | || gimple_assign_cast_p (stmt)) | |
2574 | { | |
2575 | rhs1 = gimple_assign_rhs1 (store_operand | |
2576 | ? SSA_NAME_DEF_STMT (store_operand) | |
2577 | : stmt); | |
3f5ac469 JJ |
2578 | if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR) |
2579 | rhs1 = TREE_OPERAND (rhs1, 0); | |
a9d6c7fb JJ |
2580 | /* Optimize mergeable ops ending with widening cast to _BitInt |
2581 | (or followed by store). We can lower just the limbs of the | |
2582 | cast operand and widen afterwards. */ | |
2583 | if (TREE_CODE (rhs1) == SSA_NAME | |
2584 | && (m_names == NULL | |
2585 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))) | |
2586 | && TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE | |
2587 | && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large | |
2588 | && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)), | |
2589 | limb_prec) < CEIL (prec, limb_prec) | |
2590 | || (kind == bitint_prec_huge | |
2591 | && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec))) | |
2592 | { | |
2593 | store_operand = rhs1; | |
2594 | prec = TYPE_PRECISION (TREE_TYPE (rhs1)); | |
2595 | kind = bitint_precision_kind (TREE_TYPE (rhs1)); | |
2596 | if (!TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
2597 | sext = true; | |
2598 | } | |
2599 | } | |
2600 | tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE; | |
2601 | if (kind == bitint_prec_large) | |
2602 | cnt = CEIL (prec, limb_prec); | |
2603 | else | |
2604 | { | |
2605 | rem = (prec % (2 * limb_prec)); | |
2606 | end = (prec - rem) / limb_prec; | |
2607 | cnt = 2 + CEIL (rem, limb_prec); | |
2608 | idx = idx_first = create_loop (size_zero_node, &idx_next); | |
2609 | } | |
2610 | ||
2611 | basic_block edge_bb = NULL; | |
2612 | if (eq_p) | |
2613 | { | |
2614 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
2615 | gsi_prev (&gsi); | |
2616 | edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
2617 | edge_bb = e->src; | |
2618 | if (kind == bitint_prec_large) | |
9a96a9e4 | 2619 | m_gsi = gsi_end_bb (edge_bb); |
a9d6c7fb JJ |
2620 | } |
2621 | else | |
2622 | m_after_stmt = stmt; | |
2623 | if (kind != bitint_prec_large) | |
2624 | m_upwards_2limb = end; | |
2625 | m_upwards = true; | |
2626 | ||
2627 | bool separate_ext | |
2628 | = (prec != (unsigned) TYPE_PRECISION (type) | |
2629 | && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) | |
2630 | > CEIL (prec, limb_prec))); | |
2631 | ||
2632 | for (unsigned i = 0; i < cnt; i++) | |
2633 | { | |
2634 | m_data_cnt = 0; | |
2635 | if (kind == bitint_prec_large) | |
2636 | idx = size_int (i); | |
2637 | else if (i >= 2) | |
2638 | idx = size_int (end + (i > 2)); | |
2639 | if (eq_p) | |
2640 | { | |
2641 | rhs1 = handle_operand (cmp_op1, idx); | |
2642 | tree rhs2 = handle_operand (cmp_op2, idx); | |
2643 | g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE); | |
2644 | insert_before (g); | |
2645 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
2646 | e1->flags = EDGE_FALSE_VALUE; | |
2647 | edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE); | |
2648 | e1->probability = profile_probability::unlikely (); | |
2649 | e2->probability = e1->probability.invert (); | |
2650 | if (i == 0) | |
2651 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
2652 | m_gsi = gsi_after_labels (e1->dest); | |
2653 | } | |
2654 | else | |
2655 | { | |
2656 | if (store_operand) | |
2657 | rhs1 = handle_operand (store_operand, idx); | |
2658 | else | |
2659 | rhs1 = handle_stmt (stmt, idx); | |
2660 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1))) | |
2661 | rhs1 = add_cast (m_limb_type, rhs1); | |
2662 | if (sext && i == cnt - 1) | |
2663 | ext = rhs1; | |
2664 | tree nidx = idx; | |
2665 | if (bo_idx) | |
2666 | { | |
2667 | if (tree_fits_uhwi_p (idx)) | |
2668 | nidx = size_int (tree_to_uhwi (idx) + bo_idx); | |
2669 | else | |
2670 | { | |
2671 | nidx = make_ssa_name (sizetype); | |
2672 | g = gimple_build_assign (nidx, PLUS_EXPR, idx, | |
2673 | size_int (bo_idx)); | |
2674 | insert_before (g); | |
2675 | } | |
2676 | } | |
2677 | bool done = false; | |
2678 | basic_block new_bb = NULL; | |
2679 | /* Handle stores into bit-fields. */ | |
2680 | if (bo_bit) | |
2681 | { | |
2682 | if (i == 0) | |
2683 | { | |
2684 | edge e2 = NULL; | |
2685 | if (kind != bitint_prec_large) | |
2686 | { | |
2687 | prepare_data_in_out (build_zero_cst (m_limb_type), | |
2688 | idx, &bf_next); | |
2689 | bf_next = m_data.pop (); | |
2690 | bf_cur = m_data.pop (); | |
2691 | g = gimple_build_cond (EQ_EXPR, idx, size_zero_node, | |
2692 | NULL_TREE, NULL_TREE); | |
2693 | edge edge_true; | |
2694 | if_then_else (g, profile_probability::unlikely (), | |
2695 | edge_true, e2); | |
2696 | new_bb = e2->dest; | |
2697 | } | |
2698 | tree ftype | |
2699 | = build_nonstandard_integer_type (limb_prec - bo_bit, 1); | |
3e8323e8 JJ |
2700 | tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs), |
2701 | limb_prec - bo_bit, | |
2702 | bo_idx * limb_prec + bo_bit); | |
a9d6c7fb JJ |
2703 | tree t = add_cast (ftype, rhs1); |
2704 | g = gimple_build_assign (bfr, t); | |
2705 | insert_before (g); | |
2706 | if (eh) | |
2707 | { | |
2708 | maybe_duplicate_eh_stmt (g, stmt); | |
2709 | if (eh_pad) | |
2710 | { | |
2711 | edge e = split_block (gsi_bb (m_gsi), g); | |
2712 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 JJ |
2713 | add_eh_edge (e->src, |
2714 | find_edge (gimple_bb (stmt), eh_pad)); | |
a9d6c7fb JJ |
2715 | } |
2716 | } | |
2717 | if (kind == bitint_prec_large) | |
2718 | { | |
2719 | bf_cur = rhs1; | |
2720 | done = true; | |
2721 | } | |
2722 | else if (e2) | |
2723 | m_gsi = gsi_after_labels (e2->src); | |
2724 | } | |
2725 | if (!done) | |
2726 | { | |
2727 | tree t1 = make_ssa_name (m_limb_type); | |
2728 | tree t2 = make_ssa_name (m_limb_type); | |
2729 | tree t3 = make_ssa_name (m_limb_type); | |
2730 | g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur, | |
2731 | build_int_cst (unsigned_type_node, | |
2732 | limb_prec - bo_bit)); | |
2733 | insert_before (g); | |
2734 | g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1, | |
2735 | build_int_cst (unsigned_type_node, | |
2736 | bo_bit)); | |
2737 | insert_before (g); | |
2738 | bf_cur = rhs1; | |
2739 | g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2); | |
2740 | insert_before (g); | |
2741 | rhs1 = t3; | |
2742 | if (bf_next && i == 1) | |
2743 | { | |
2744 | g = gimple_build_assign (bf_next, bf_cur); | |
2745 | insert_before (g); | |
2746 | } | |
2747 | } | |
2748 | } | |
2749 | if (!done) | |
2750 | { | |
2751 | /* Handle bit-field access to partial last limb if needed. */ | |
2752 | if (nlhs | |
2753 | && i == cnt - 1 | |
2754 | && !separate_ext | |
2755 | && tree_fits_uhwi_p (idx)) | |
2756 | { | |
2757 | unsigned int tprec = TYPE_PRECISION (type); | |
982250b2 | 2758 | unsigned int rprec = (tprec - 1) % limb_prec + 1; |
a9d6c7fb JJ |
2759 | if (rprec + bo_bit < (unsigned) limb_prec) |
2760 | { | |
2761 | tree ftype | |
2762 | = build_nonstandard_integer_type (rprec + bo_bit, 1); | |
3e8323e8 JJ |
2763 | tree bfr |
2764 | = build_bit_field_ref (ftype, unshare_expr (nlhs), | |
2765 | rprec + bo_bit, | |
2766 | (bo_idx + tprec / limb_prec) | |
2767 | * limb_prec); | |
a9d6c7fb JJ |
2768 | tree t = add_cast (ftype, rhs1); |
2769 | g = gimple_build_assign (bfr, t); | |
2770 | done = true; | |
2771 | bf_cur = NULL_TREE; | |
2772 | } | |
2773 | else if (rprec + bo_bit == (unsigned) limb_prec) | |
2774 | bf_cur = NULL_TREE; | |
2775 | } | |
2776 | /* Otherwise, stores to any other lhs. */ | |
2777 | if (!done) | |
2778 | { | |
0613b12d JJ |
2779 | tree l = limb_access (nlhs ? NULL_TREE : lhs_type, |
2780 | nlhs ? nlhs : lhs, nidx, true); | |
a9d6c7fb JJ |
2781 | g = gimple_build_assign (l, rhs1); |
2782 | } | |
2783 | insert_before (g); | |
2784 | if (eh) | |
2785 | { | |
2786 | maybe_duplicate_eh_stmt (g, stmt); | |
2787 | if (eh_pad) | |
2788 | { | |
2789 | edge e = split_block (gsi_bb (m_gsi), g); | |
2790 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 JJ |
2791 | add_eh_edge (e->src, |
2792 | find_edge (gimple_bb (stmt), eh_pad)); | |
a9d6c7fb JJ |
2793 | } |
2794 | } | |
2795 | if (new_bb) | |
2796 | m_gsi = gsi_after_labels (new_bb); | |
2797 | } | |
2798 | } | |
2799 | m_first = false; | |
2800 | if (kind == bitint_prec_huge && i <= 1) | |
2801 | { | |
2802 | if (i == 0) | |
2803 | { | |
2804 | idx = make_ssa_name (sizetype); | |
2805 | g = gimple_build_assign (idx, PLUS_EXPR, idx_first, | |
2806 | size_one_node); | |
2807 | insert_before (g); | |
2808 | } | |
2809 | else | |
2810 | { | |
2811 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first, | |
2812 | size_int (2)); | |
2813 | insert_before (g); | |
2814 | g = gimple_build_cond (NE_EXPR, idx_next, size_int (end), | |
2815 | NULL_TREE, NULL_TREE); | |
2816 | insert_before (g); | |
2817 | if (eq_p) | |
2818 | m_gsi = gsi_after_labels (edge_bb); | |
2819 | else | |
2820 | m_gsi = gsi_for_stmt (stmt); | |
be9e8de6 | 2821 | m_bb = NULL; |
a9d6c7fb JJ |
2822 | } |
2823 | } | |
2824 | } | |
2825 | ||
2826 | if (separate_ext) | |
2827 | { | |
2828 | if (sext) | |
2829 | { | |
2830 | ext = add_cast (signed_type_for (m_limb_type), ext); | |
2831 | tree lpm1 = build_int_cst (unsigned_type_node, | |
2832 | limb_prec - 1); | |
2833 | tree n = make_ssa_name (TREE_TYPE (ext)); | |
2834 | g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1); | |
2835 | insert_before (g); | |
2836 | ext = add_cast (m_limb_type, n); | |
2837 | } | |
2838 | else | |
2839 | ext = build_zero_cst (m_limb_type); | |
2840 | kind = bitint_precision_kind (type); | |
2841 | unsigned start = CEIL (prec, limb_prec); | |
2842 | prec = TYPE_PRECISION (type); | |
2843 | idx = idx_first = idx_next = NULL_TREE; | |
2844 | if (prec <= (start + 2 + (bo_bit != 0)) * limb_prec) | |
2845 | kind = bitint_prec_large; | |
2846 | if (kind == bitint_prec_large) | |
2847 | cnt = CEIL (prec, limb_prec) - start; | |
2848 | else | |
2849 | { | |
2850 | rem = prec % limb_prec; | |
2851 | end = (prec - rem) / limb_prec; | |
2852 | cnt = (bo_bit != 0) + 1 + (rem != 0); | |
2853 | } | |
2854 | for (unsigned i = 0; i < cnt; i++) | |
2855 | { | |
2856 | if (kind == bitint_prec_large || (i == 0 && bo_bit != 0)) | |
2857 | idx = size_int (start + i); | |
0ca64f84 | 2858 | else if (i == cnt - 1 && (rem != 0)) |
a9d6c7fb JJ |
2859 | idx = size_int (end); |
2860 | else if (i == (bo_bit != 0)) | |
2861 | idx = create_loop (size_int (start + i), &idx_next); | |
2862 | rhs1 = ext; | |
2863 | if (bf_cur != NULL_TREE && bf_cur != ext) | |
2864 | { | |
2865 | tree t1 = make_ssa_name (m_limb_type); | |
2866 | g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur, | |
2867 | build_int_cst (unsigned_type_node, | |
2868 | limb_prec - bo_bit)); | |
2869 | insert_before (g); | |
2870 | if (integer_zerop (ext)) | |
2871 | rhs1 = t1; | |
2872 | else | |
2873 | { | |
2874 | tree t2 = make_ssa_name (m_limb_type); | |
2875 | rhs1 = make_ssa_name (m_limb_type); | |
2876 | g = gimple_build_assign (t2, LSHIFT_EXPR, ext, | |
2877 | build_int_cst (unsigned_type_node, | |
2878 | bo_bit)); | |
2879 | insert_before (g); | |
2880 | g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2); | |
2881 | insert_before (g); | |
2882 | } | |
2883 | bf_cur = ext; | |
2884 | } | |
2885 | tree nidx = idx; | |
2886 | if (bo_idx) | |
2887 | { | |
2888 | if (tree_fits_uhwi_p (idx)) | |
2889 | nidx = size_int (tree_to_uhwi (idx) + bo_idx); | |
2890 | else | |
2891 | { | |
2892 | nidx = make_ssa_name (sizetype); | |
2893 | g = gimple_build_assign (nidx, PLUS_EXPR, idx, | |
2894 | size_int (bo_idx)); | |
2895 | insert_before (g); | |
2896 | } | |
2897 | } | |
2898 | bool done = false; | |
2899 | /* Handle bit-field access to partial last limb if needed. */ | |
2900 | if (nlhs && i == cnt - 1) | |
2901 | { | |
2902 | unsigned int tprec = TYPE_PRECISION (type); | |
982250b2 | 2903 | unsigned int rprec = (tprec - 1) % limb_prec + 1; |
a9d6c7fb JJ |
2904 | if (rprec + bo_bit < (unsigned) limb_prec) |
2905 | { | |
2906 | tree ftype | |
2907 | = build_nonstandard_integer_type (rprec + bo_bit, 1); | |
3e8323e8 JJ |
2908 | tree bfr |
2909 | = build_bit_field_ref (ftype, unshare_expr (nlhs), | |
2910 | rprec + bo_bit, | |
2911 | (bo_idx + tprec / limb_prec) | |
2912 | * limb_prec); | |
a9d6c7fb JJ |
2913 | tree t = add_cast (ftype, rhs1); |
2914 | g = gimple_build_assign (bfr, t); | |
2915 | done = true; | |
2916 | bf_cur = NULL_TREE; | |
2917 | } | |
2918 | else if (rprec + bo_bit == (unsigned) limb_prec) | |
2919 | bf_cur = NULL_TREE; | |
2920 | } | |
2921 | /* Otherwise, stores to any other lhs. */ | |
2922 | if (!done) | |
2923 | { | |
0613b12d JJ |
2924 | tree l = limb_access (nlhs ? NULL_TREE : lhs_type, |
2925 | nlhs ? nlhs : lhs, nidx, true); | |
a9d6c7fb JJ |
2926 | g = gimple_build_assign (l, rhs1); |
2927 | } | |
2928 | insert_before (g); | |
2929 | if (eh) | |
2930 | { | |
2931 | maybe_duplicate_eh_stmt (g, stmt); | |
2932 | if (eh_pad) | |
2933 | { | |
2934 | edge e = split_block (gsi_bb (m_gsi), g); | |
2935 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 | 2936 | add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad)); |
a9d6c7fb JJ |
2937 | } |
2938 | } | |
2939 | if (kind == bitint_prec_huge && i == (bo_bit != 0)) | |
2940 | { | |
2941 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, | |
2942 | size_one_node); | |
2943 | insert_before (g); | |
2944 | g = gimple_build_cond (NE_EXPR, idx_next, size_int (end), | |
2945 | NULL_TREE, NULL_TREE); | |
2946 | insert_before (g); | |
2947 | m_gsi = gsi_for_stmt (stmt); | |
be9e8de6 | 2948 | m_bb = NULL; |
a9d6c7fb JJ |
2949 | } |
2950 | } | |
2951 | } | |
2952 | if (bf_cur != NULL_TREE) | |
2953 | { | |
2954 | unsigned int tprec = TYPE_PRECISION (type); | |
982250b2 JJ |
2955 | unsigned int rprec = (tprec + bo_bit) % limb_prec; |
2956 | tree ftype = build_nonstandard_integer_type (rprec, 1); | |
3e8323e8 | 2957 | tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs), |
982250b2 JJ |
2958 | rprec, |
2959 | (bo_idx + (tprec + bo_bit) / limb_prec) | |
3e8323e8 | 2960 | * limb_prec); |
a9d6c7fb JJ |
2961 | rhs1 = bf_cur; |
2962 | if (bf_cur != ext) | |
2963 | { | |
2964 | rhs1 = make_ssa_name (TREE_TYPE (rhs1)); | |
2965 | g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur, | |
2966 | build_int_cst (unsigned_type_node, | |
2967 | limb_prec - bo_bit)); | |
2968 | insert_before (g); | |
2969 | } | |
2970 | rhs1 = add_cast (ftype, rhs1); | |
2971 | g = gimple_build_assign (bfr, rhs1); | |
2972 | insert_before (g); | |
2973 | if (eh) | |
2974 | { | |
2975 | maybe_duplicate_eh_stmt (g, stmt); | |
2976 | if (eh_pad) | |
2977 | { | |
2978 | edge e = split_block (gsi_bb (m_gsi), g); | |
2979 | m_gsi = gsi_after_labels (e->dest); | |
6c124873 | 2980 | add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad)); |
a9d6c7fb JJ |
2981 | } |
2982 | } | |
2983 | } | |
2984 | ||
2985 | if (gimple_store_p (stmt)) | |
2986 | { | |
2987 | unlink_stmt_vdef (stmt); | |
2988 | release_ssa_name (gimple_vdef (stmt)); | |
2989 | gsi_remove (&m_gsi, true); | |
2990 | } | |
2991 | if (eq_p) | |
2992 | { | |
2993 | lhs = make_ssa_name (boolean_type_node); | |
2994 | basic_block bb = gimple_bb (stmt); | |
2995 | gphi *phi = create_phi_node (lhs, bb); | |
2996 | edge e = find_edge (gsi_bb (m_gsi), bb); | |
2997 | unsigned int n = EDGE_COUNT (bb->preds); | |
2998 | for (unsigned int i = 0; i < n; i++) | |
2999 | { | |
3000 | edge e2 = EDGE_PRED (bb, i); | |
3001 | add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node, | |
3002 | e2, UNKNOWN_LOCATION); | |
3003 | } | |
3004 | cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR; | |
3005 | return lhs; | |
3006 | } | |
3007 | else | |
3008 | return NULL_TREE; | |
3009 | } | |
3010 | ||
3011 | /* Handle a large/huge _BitInt comparison statement STMT other than | |
3012 | EQ_EXPR/NE_EXPR. CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in | |
3013 | lower_mergeable_stmt. The {GT,GE,LT,LE}_EXPR comparisons are | |
3014 | lowered by iteration from the most significant limb downwards to | |
3015 | the least significant one, for large _BitInt in straight line code, | |
3016 | otherwise with most significant limb handled in | |
3017 | straight line code followed by a loop handling one limb at a time. | |
3018 | Comparisons with unsigned huge _BitInt with precisions which are | |
3019 | multiples of limb precision can use just the loop and don't need to | |
3020 | handle most significant limb before the loop. The loop or straight | |
3021 | line code jumps to final basic block if a particular pair of limbs | |
3022 | is not equal. */ | |
3023 | ||
3024 | tree | |
3025 | bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code, | |
3026 | tree cmp_op1, tree cmp_op2) | |
3027 | { | |
3028 | tree type = TREE_TYPE (cmp_op1); | |
3029 | gcc_assert (TREE_CODE (type) == BITINT_TYPE); | |
3030 | bitint_prec_kind kind = bitint_precision_kind (type); | |
3031 | gcc_assert (kind >= bitint_prec_large); | |
3032 | gimple *g; | |
3033 | if (!TYPE_UNSIGNED (type) | |
3034 | && integer_zerop (cmp_op2) | |
3035 | && (cmp_code == GE_EXPR || cmp_code == LT_EXPR)) | |
3036 | { | |
3037 | unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1; | |
3038 | tree idx = size_int (end); | |
3039 | m_data_cnt = 0; | |
3040 | tree rhs1 = handle_operand (cmp_op1, idx); | |
3041 | if (TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
3042 | { | |
3043 | tree stype = signed_type_for (TREE_TYPE (rhs1)); | |
3044 | rhs1 = add_cast (stype, rhs1); | |
3045 | } | |
3046 | tree lhs = make_ssa_name (boolean_type_node); | |
3047 | g = gimple_build_assign (lhs, cmp_code, rhs1, | |
3048 | build_zero_cst (TREE_TYPE (rhs1))); | |
3049 | insert_before (g); | |
3050 | cmp_code = NE_EXPR; | |
3051 | return lhs; | |
3052 | } | |
3053 | ||
3054 | unsigned cnt, rem = 0, end = 0; | |
3055 | tree idx = NULL_TREE, idx_next = NULL_TREE; | |
3056 | if (kind == bitint_prec_large) | |
3057 | cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec); | |
3058 | else | |
3059 | { | |
3060 | rem = ((unsigned) TYPE_PRECISION (type) % limb_prec); | |
3061 | if (rem == 0 && !TYPE_UNSIGNED (type)) | |
3062 | rem = limb_prec; | |
3063 | end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec; | |
3064 | cnt = 1 + (rem != 0); | |
3065 | } | |
3066 | ||
3067 | basic_block edge_bb = NULL; | |
3068 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
3069 | gsi_prev (&gsi); | |
3070 | edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
3071 | edge_bb = e->src; | |
9a96a9e4 | 3072 | m_gsi = gsi_end_bb (edge_bb); |
a9d6c7fb JJ |
3073 | |
3074 | edge *edges = XALLOCAVEC (edge, cnt * 2); | |
3075 | for (unsigned i = 0; i < cnt; i++) | |
3076 | { | |
3077 | m_data_cnt = 0; | |
3078 | if (kind == bitint_prec_large) | |
3079 | idx = size_int (cnt - i - 1); | |
3080 | else if (i == cnt - 1) | |
3081 | idx = create_loop (size_int (end - 1), &idx_next); | |
3082 | else | |
3083 | idx = size_int (end); | |
3084 | tree rhs1 = handle_operand (cmp_op1, idx); | |
3085 | tree rhs2 = handle_operand (cmp_op2, idx); | |
3086 | if (i == 0 | |
3087 | && !TYPE_UNSIGNED (type) | |
3088 | && TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
3089 | { | |
3090 | tree stype = signed_type_for (TREE_TYPE (rhs1)); | |
3091 | rhs1 = add_cast (stype, rhs1); | |
3092 | rhs2 = add_cast (stype, rhs2); | |
3093 | } | |
3094 | g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE); | |
3095 | insert_before (g); | |
3096 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
3097 | e1->flags = EDGE_FALSE_VALUE; | |
3098 | edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE); | |
3099 | e1->probability = profile_probability::likely (); | |
3100 | e2->probability = e1->probability.invert (); | |
3101 | if (i == 0) | |
3102 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
3103 | m_gsi = gsi_after_labels (e1->dest); | |
3104 | edges[2 * i] = e2; | |
3105 | g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE); | |
3106 | insert_before (g); | |
3107 | e1 = split_block (gsi_bb (m_gsi), g); | |
3108 | e1->flags = EDGE_FALSE_VALUE; | |
3109 | e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE); | |
3110 | e1->probability = profile_probability::unlikely (); | |
3111 | e2->probability = e1->probability.invert (); | |
3112 | m_gsi = gsi_after_labels (e1->dest); | |
3113 | edges[2 * i + 1] = e2; | |
3114 | m_first = false; | |
3115 | if (kind == bitint_prec_huge && i == cnt - 1) | |
3116 | { | |
3117 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1)); | |
3118 | insert_before (g); | |
3119 | g = gimple_build_cond (NE_EXPR, idx, size_zero_node, | |
3120 | NULL_TREE, NULL_TREE); | |
3121 | insert_before (g); | |
3122 | edge true_edge, false_edge; | |
3123 | extract_true_false_edges_from_block (gsi_bb (m_gsi), | |
3124 | &true_edge, &false_edge); | |
3125 | m_gsi = gsi_after_labels (false_edge->dest); | |
be9e8de6 | 3126 | m_bb = NULL; |
a9d6c7fb JJ |
3127 | } |
3128 | } | |
3129 | ||
3130 | tree lhs = make_ssa_name (boolean_type_node); | |
3131 | basic_block bb = gimple_bb (stmt); | |
3132 | gphi *phi = create_phi_node (lhs, bb); | |
3133 | for (unsigned int i = 0; i < cnt * 2; i++) | |
3134 | { | |
3135 | tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR) | |
3136 | ^ (i & 1)) ? boolean_true_node : boolean_false_node; | |
3137 | add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION); | |
3138 | } | |
3139 | add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR) | |
3140 | ? boolean_true_node : boolean_false_node, | |
3141 | find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION); | |
3142 | cmp_code = NE_EXPR; | |
3143 | return lhs; | |
3144 | } | |
3145 | ||
3146 | /* Lower large/huge _BitInt left and right shift except for left | |
3147 | shift by < limb_prec constant. */ | |
3148 | ||
3149 | void | |
3150 | bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt) | |
3151 | { | |
3152 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
3153 | tree lhs = gimple_assign_lhs (stmt); | |
3154 | tree_code rhs_code = gimple_assign_rhs_code (stmt); | |
3155 | tree type = TREE_TYPE (rhs1); | |
3156 | gimple *final_stmt = gsi_stmt (m_gsi); | |
3157 | gcc_assert (TREE_CODE (type) == BITINT_TYPE | |
3158 | && bitint_precision_kind (type) >= bitint_prec_large); | |
3159 | int prec = TYPE_PRECISION (type); | |
3160 | tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4; | |
3161 | gimple *g; | |
3162 | if (obj == NULL_TREE) | |
3163 | { | |
3164 | int part = var_to_partition (m_map, lhs); | |
3165 | gcc_assert (m_vars[part] != NULL_TREE); | |
3166 | obj = m_vars[part]; | |
3167 | } | |
3168 | /* Preparation code common for both left and right shifts. | |
3169 | unsigned n1 = n % limb_prec; | |
3170 | size_t n2 = n / limb_prec; | |
3171 | size_t n3 = n1 != 0; | |
3172 | unsigned n4 = (limb_prec - n1) % limb_prec; | |
3173 | (for power of 2 limb_prec n4 can be -n1 & (limb_prec)). */ | |
3174 | if (TREE_CODE (n) == INTEGER_CST) | |
3175 | { | |
3176 | tree lp = build_int_cst (TREE_TYPE (n), limb_prec); | |
3177 | n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp); | |
3178 | n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp)); | |
3179 | n3 = size_int (!integer_zerop (n1)); | |
3180 | n4 = int_const_binop (TRUNC_MOD_EXPR, | |
3181 | int_const_binop (MINUS_EXPR, lp, n1), lp); | |
3182 | } | |
3183 | else | |
3184 | { | |
3185 | n1 = make_ssa_name (TREE_TYPE (n)); | |
3186 | n2 = make_ssa_name (sizetype); | |
3187 | n3 = make_ssa_name (sizetype); | |
3188 | n4 = make_ssa_name (TREE_TYPE (n)); | |
3189 | if (pow2p_hwi (limb_prec)) | |
3190 | { | |
3191 | tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1); | |
3192 | g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1); | |
3193 | insert_before (g); | |
3194 | g = gimple_build_assign (useless_type_conversion_p (sizetype, | |
3195 | TREE_TYPE (n)) | |
3196 | ? n2 : make_ssa_name (TREE_TYPE (n)), | |
3197 | RSHIFT_EXPR, n, | |
3198 | build_int_cst (TREE_TYPE (n), | |
3199 | exact_log2 (limb_prec))); | |
3200 | insert_before (g); | |
3201 | if (gimple_assign_lhs (g) != n2) | |
3202 | { | |
3203 | g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g)); | |
3204 | insert_before (g); | |
3205 | } | |
3206 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)), | |
3207 | NEGATE_EXPR, n1); | |
3208 | insert_before (g); | |
3209 | g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g), | |
3210 | lpm1); | |
3211 | insert_before (g); | |
3212 | } | |
3213 | else | |
3214 | { | |
3215 | tree lp = build_int_cst (TREE_TYPE (n), limb_prec); | |
3216 | g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp); | |
3217 | insert_before (g); | |
3218 | g = gimple_build_assign (useless_type_conversion_p (sizetype, | |
3219 | TREE_TYPE (n)) | |
3220 | ? n2 : make_ssa_name (TREE_TYPE (n)), | |
3221 | TRUNC_DIV_EXPR, n, lp); | |
3222 | insert_before (g); | |
3223 | if (gimple_assign_lhs (g) != n2) | |
3224 | { | |
3225 | g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g)); | |
3226 | insert_before (g); | |
3227 | } | |
3228 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)), | |
3229 | MINUS_EXPR, lp, n1); | |
3230 | insert_before (g); | |
3231 | g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g), | |
3232 | lp); | |
3233 | insert_before (g); | |
3234 | } | |
3235 | g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1, | |
3236 | build_zero_cst (TREE_TYPE (n))); | |
3237 | insert_before (g); | |
3238 | g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g)); | |
3239 | insert_before (g); | |
3240 | } | |
3241 | tree p = build_int_cst (sizetype, | |
3242 | prec / limb_prec - (prec % limb_prec == 0)); | |
3243 | if (rhs_code == RSHIFT_EXPR) | |
3244 | { | |
3245 | /* Lower | |
3246 | dst = src >> n; | |
3247 | as | |
3248 | unsigned n1 = n % limb_prec; | |
3249 | size_t n2 = n / limb_prec; | |
3250 | size_t n3 = n1 != 0; | |
3251 | unsigned n4 = (limb_prec - n1) % limb_prec; | |
3252 | size_t idx; | |
3253 | size_t p = prec / limb_prec - (prec % limb_prec == 0); | |
3254 | int signed_p = (typeof (src) -1) < 0; | |
3255 | for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0)) | |
3256 | ? p : p - n3); ++idx) | |
3257 | dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4); | |
3258 | limb_type ext; | |
3259 | if (prec % limb_prec == 0) | |
3260 | ext = src[p]; | |
3261 | else if (signed_p) | |
3262 | ext = ((signed limb_type) (src[p] << (limb_prec | |
3263 | - (prec % limb_prec)))) | |
3264 | >> (limb_prec - (prec % limb_prec)); | |
3265 | else | |
3266 | ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1); | |
3267 | if (!signed_p && (prec % limb_prec == 0)) | |
3268 | ; | |
3269 | else if (idx < prec / 64) | |
3270 | { | |
3271 | dst[idx - n2] = (src[idx] >> n1) | (ext << n4); | |
3272 | ++idx; | |
3273 | } | |
3274 | idx -= n2; | |
3275 | if (signed_p) | |
3276 | { | |
3277 | dst[idx] = ((signed limb_type) ext) >> n1; | |
3278 | ext = ((signed limb_type) ext) >> (limb_prec - 1); | |
3279 | } | |
3280 | else | |
3281 | { | |
3282 | dst[idx] = ext >> n1; | |
3283 | ext = 0; | |
3284 | } | |
3285 | for (++idx; idx <= p; ++idx) | |
3286 | dst[idx] = ext; */ | |
3287 | tree pmn3; | |
3288 | if (TYPE_UNSIGNED (type) && prec % limb_prec == 0) | |
3289 | pmn3 = p; | |
3290 | else if (TREE_CODE (n3) == INTEGER_CST) | |
3291 | pmn3 = int_const_binop (MINUS_EXPR, p, n3); | |
3292 | else | |
3293 | { | |
3294 | pmn3 = make_ssa_name (sizetype); | |
3295 | g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3); | |
3296 | insert_before (g); | |
3297 | } | |
3298 | g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE); | |
3299 | edge edge_true, edge_false; | |
3300 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3301 | tree idx_next; | |
3302 | tree idx = create_loop (n2, &idx_next); | |
3303 | tree idxmn2 = make_ssa_name (sizetype); | |
3304 | tree idxpn3 = make_ssa_name (sizetype); | |
3305 | g = gimple_build_assign (idxmn2, MINUS_EXPR, idx, n2); | |
3306 | insert_before (g); | |
3307 | g = gimple_build_assign (idxpn3, PLUS_EXPR, idx, n3); | |
3308 | insert_before (g); | |
3309 | m_data_cnt = 0; | |
3310 | tree t1 = handle_operand (rhs1, idx); | |
3311 | m_first = false; | |
3312 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3313 | RSHIFT_EXPR, t1, n1); | |
3314 | insert_before (g); | |
3315 | t1 = gimple_assign_lhs (g); | |
3316 | if (!integer_zerop (n3)) | |
3317 | { | |
3318 | m_data_cnt = 0; | |
3319 | tree t2 = handle_operand (rhs1, idxpn3); | |
3320 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3321 | LSHIFT_EXPR, t2, n4); | |
3322 | insert_before (g); | |
3323 | t2 = gimple_assign_lhs (g); | |
3324 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3325 | BIT_IOR_EXPR, t1, t2); | |
3326 | insert_before (g); | |
3327 | t1 = gimple_assign_lhs (g); | |
3328 | } | |
3329 | tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true); | |
3330 | g = gimple_build_assign (l, t1); | |
3331 | insert_before (g); | |
3332 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node); | |
3333 | insert_before (g); | |
3334 | g = gimple_build_cond (LT_EXPR, idx_next, pmn3, NULL_TREE, NULL_TREE); | |
3335 | insert_before (g); | |
3336 | idx = make_ssa_name (sizetype); | |
3337 | m_gsi = gsi_for_stmt (final_stmt); | |
3338 | gphi *phi = create_phi_node (idx, gsi_bb (m_gsi)); | |
3339 | edge_false = find_edge (edge_false->src, gsi_bb (m_gsi)); | |
3340 | edge_true = EDGE_PRED (gsi_bb (m_gsi), | |
3341 | EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false); | |
3342 | add_phi_arg (phi, n2, edge_false, UNKNOWN_LOCATION); | |
3343 | add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION); | |
3344 | m_data_cnt = 0; | |
3345 | tree ms = handle_operand (rhs1, p); | |
3346 | tree ext = ms; | |
3347 | if (!types_compatible_p (TREE_TYPE (ms), m_limb_type)) | |
3348 | ext = add_cast (m_limb_type, ms); | |
3349 | if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0) | |
3350 | && !integer_zerop (n3)) | |
3351 | { | |
3352 | g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE); | |
3353 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3354 | m_data_cnt = 0; | |
3355 | t1 = handle_operand (rhs1, idx); | |
3356 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3357 | RSHIFT_EXPR, t1, n1); | |
3358 | insert_before (g); | |
3359 | t1 = gimple_assign_lhs (g); | |
3360 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3361 | LSHIFT_EXPR, ext, n4); | |
3362 | insert_before (g); | |
3363 | tree t2 = gimple_assign_lhs (g); | |
3364 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3365 | BIT_IOR_EXPR, t1, t2); | |
3366 | insert_before (g); | |
3367 | t1 = gimple_assign_lhs (g); | |
3368 | idxmn2 = make_ssa_name (sizetype); | |
3369 | g = gimple_build_assign (idxmn2, MINUS_EXPR, idx, n2); | |
3370 | insert_before (g); | |
3371 | l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true); | |
3372 | g = gimple_build_assign (l, t1); | |
3373 | insert_before (g); | |
3374 | idx_next = make_ssa_name (sizetype); | |
3375 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node); | |
3376 | insert_before (g); | |
3377 | m_gsi = gsi_for_stmt (final_stmt); | |
3378 | tree nidx = make_ssa_name (sizetype); | |
3379 | phi = create_phi_node (nidx, gsi_bb (m_gsi)); | |
3380 | edge_false = find_edge (edge_false->src, gsi_bb (m_gsi)); | |
3381 | edge_true = EDGE_PRED (gsi_bb (m_gsi), | |
3382 | EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false); | |
3383 | add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION); | |
3384 | add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION); | |
3385 | idx = nidx; | |
3386 | } | |
3387 | g = gimple_build_assign (make_ssa_name (sizetype), MINUS_EXPR, idx, n2); | |
3388 | insert_before (g); | |
3389 | idx = gimple_assign_lhs (g); | |
3390 | tree sext = ext; | |
3391 | if (!TYPE_UNSIGNED (type)) | |
3392 | sext = add_cast (signed_type_for (m_limb_type), ext); | |
3393 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)), | |
3394 | RSHIFT_EXPR, sext, n1); | |
3395 | insert_before (g); | |
3396 | t1 = gimple_assign_lhs (g); | |
3397 | if (!TYPE_UNSIGNED (type)) | |
3398 | { | |
3399 | t1 = add_cast (m_limb_type, t1); | |
3400 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)), | |
3401 | RSHIFT_EXPR, sext, | |
3402 | build_int_cst (TREE_TYPE (n), | |
3403 | limb_prec - 1)); | |
3404 | insert_before (g); | |
3405 | ext = add_cast (m_limb_type, gimple_assign_lhs (g)); | |
3406 | } | |
3407 | else | |
3408 | ext = build_zero_cst (m_limb_type); | |
3409 | l = limb_access (TREE_TYPE (lhs), obj, idx, true); | |
3410 | g = gimple_build_assign (l, t1); | |
3411 | insert_before (g); | |
3412 | g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx, | |
3413 | size_one_node); | |
3414 | insert_before (g); | |
3415 | idx = gimple_assign_lhs (g); | |
3416 | g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE); | |
3417 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3418 | idx = create_loop (idx, &idx_next); | |
3419 | l = limb_access (TREE_TYPE (lhs), obj, idx, true); | |
3420 | g = gimple_build_assign (l, ext); | |
3421 | insert_before (g); | |
3422 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node); | |
3423 | insert_before (g); | |
3424 | g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE); | |
3425 | insert_before (g); | |
3426 | } | |
3427 | else | |
3428 | { | |
3429 | /* Lower | |
3430 | dst = src << n; | |
3431 | as | |
3432 | unsigned n1 = n % limb_prec; | |
3433 | size_t n2 = n / limb_prec; | |
3434 | size_t n3 = n1 != 0; | |
3435 | unsigned n4 = (limb_prec - n1) % limb_prec; | |
3436 | size_t idx; | |
3437 | size_t p = prec / limb_prec - (prec % limb_prec == 0); | |
3438 | for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx) | |
3439 | dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4); | |
3440 | if (n1) | |
3441 | { | |
3442 | dst[idx] = src[idx - n2] << n1; | |
3443 | --idx; | |
3444 | } | |
3445 | for (; (ssize_t) idx >= 0; --idx) | |
3446 | dst[idx] = 0; */ | |
3447 | tree n2pn3; | |
3448 | if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST) | |
3449 | n2pn3 = int_const_binop (PLUS_EXPR, n2, n3); | |
3450 | else | |
3451 | { | |
3452 | n2pn3 = make_ssa_name (sizetype); | |
3453 | g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3); | |
3454 | insert_before (g); | |
3455 | } | |
3456 | /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST | |
3457 | idx even to access the most significant partial limb. */ | |
3458 | m_var_msb = true; | |
3459 | if (integer_zerop (n3)) | |
3460 | /* For n3 == 0 p >= n2 + n3 is always true for all valid shift | |
3461 | counts. Emit if (true) condition that can be optimized later. */ | |
3462 | g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node, | |
3463 | NULL_TREE, NULL_TREE); | |
3464 | else | |
3465 | g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE); | |
3466 | edge edge_true, edge_false; | |
3467 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3468 | tree idx_next; | |
3469 | tree idx = create_loop (p, &idx_next); | |
3470 | tree idxmn2 = make_ssa_name (sizetype); | |
3471 | tree idxmn2mn3 = make_ssa_name (sizetype); | |
3472 | g = gimple_build_assign (idxmn2, MINUS_EXPR, idx, n2); | |
3473 | insert_before (g); | |
3474 | g = gimple_build_assign (idxmn2mn3, MINUS_EXPR, idxmn2, n3); | |
3475 | insert_before (g); | |
3476 | m_data_cnt = 0; | |
3477 | tree t1 = handle_operand (rhs1, idxmn2); | |
3478 | m_first = false; | |
3479 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3480 | LSHIFT_EXPR, t1, n1); | |
3481 | insert_before (g); | |
3482 | t1 = gimple_assign_lhs (g); | |
3483 | if (!integer_zerop (n3)) | |
3484 | { | |
3485 | m_data_cnt = 0; | |
3486 | tree t2 = handle_operand (rhs1, idxmn2mn3); | |
3487 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3488 | RSHIFT_EXPR, t2, n4); | |
3489 | insert_before (g); | |
3490 | t2 = gimple_assign_lhs (g); | |
3491 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3492 | BIT_IOR_EXPR, t1, t2); | |
3493 | insert_before (g); | |
3494 | t1 = gimple_assign_lhs (g); | |
3495 | } | |
3496 | tree l = limb_access (TREE_TYPE (lhs), obj, idx, true); | |
3497 | g = gimple_build_assign (l, t1); | |
3498 | insert_before (g); | |
3499 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1)); | |
3500 | insert_before (g); | |
3501 | tree sn2pn3 = add_cast (ssizetype, n2pn3); | |
3502 | g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next), sn2pn3, | |
3503 | NULL_TREE, NULL_TREE); | |
3504 | insert_before (g); | |
3505 | idx = make_ssa_name (sizetype); | |
3506 | m_gsi = gsi_for_stmt (final_stmt); | |
3507 | gphi *phi = create_phi_node (idx, gsi_bb (m_gsi)); | |
3508 | edge_false = find_edge (edge_false->src, gsi_bb (m_gsi)); | |
3509 | edge_true = EDGE_PRED (gsi_bb (m_gsi), | |
3510 | EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false); | |
3511 | add_phi_arg (phi, p, edge_false, UNKNOWN_LOCATION); | |
3512 | add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION); | |
3513 | m_data_cnt = 0; | |
3514 | if (!integer_zerop (n3)) | |
3515 | { | |
3516 | g = gimple_build_cond (NE_EXPR, n3, size_zero_node, | |
3517 | NULL_TREE, NULL_TREE); | |
3518 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3519 | idxmn2 = make_ssa_name (sizetype); | |
3520 | g = gimple_build_assign (idxmn2, MINUS_EXPR, idx, n2); | |
3521 | insert_before (g); | |
3522 | m_data_cnt = 0; | |
3523 | t1 = handle_operand (rhs1, idxmn2); | |
3524 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3525 | LSHIFT_EXPR, t1, n1); | |
3526 | insert_before (g); | |
3527 | t1 = gimple_assign_lhs (g); | |
3528 | l = limb_access (TREE_TYPE (lhs), obj, idx, true); | |
3529 | g = gimple_build_assign (l, t1); | |
3530 | insert_before (g); | |
3531 | idx_next = make_ssa_name (sizetype); | |
3532 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1)); | |
3533 | insert_before (g); | |
3534 | m_gsi = gsi_for_stmt (final_stmt); | |
3535 | tree nidx = make_ssa_name (sizetype); | |
3536 | phi = create_phi_node (nidx, gsi_bb (m_gsi)); | |
3537 | edge_false = find_edge (edge_false->src, gsi_bb (m_gsi)); | |
3538 | edge_true = EDGE_PRED (gsi_bb (m_gsi), | |
3539 | EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false); | |
3540 | add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION); | |
3541 | add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION); | |
3542 | idx = nidx; | |
3543 | } | |
3544 | g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx), | |
3545 | ssize_int (0), NULL_TREE, NULL_TREE); | |
3546 | if_then (g, profile_probability::likely (), edge_true, edge_false); | |
3547 | idx = create_loop (idx, &idx_next); | |
3548 | l = limb_access (TREE_TYPE (lhs), obj, idx, true); | |
3549 | g = gimple_build_assign (l, build_zero_cst (m_limb_type)); | |
3550 | insert_before (g); | |
3551 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1)); | |
3552 | insert_before (g); | |
3553 | g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next), | |
3554 | ssize_int (0), NULL_TREE, NULL_TREE); | |
3555 | insert_before (g); | |
3556 | } | |
3557 | } | |
3558 | ||
3559 | /* Lower large/huge _BitInt multiplication or division. */ | |
3560 | ||
3561 | void | |
3562 | bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt) | |
3563 | { | |
3564 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
3565 | tree rhs2 = gimple_assign_rhs2 (stmt); | |
3566 | tree lhs = gimple_assign_lhs (stmt); | |
3567 | tree_code rhs_code = gimple_assign_rhs_code (stmt); | |
3568 | tree type = TREE_TYPE (rhs1); | |
3569 | gcc_assert (TREE_CODE (type) == BITINT_TYPE | |
3570 | && bitint_precision_kind (type) >= bitint_prec_large); | |
3571 | int prec = TYPE_PRECISION (type), prec1, prec2; | |
3572 | rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1); | |
3573 | rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2); | |
3574 | if (obj == NULL_TREE) | |
3575 | { | |
3576 | int part = var_to_partition (m_map, lhs); | |
3577 | gcc_assert (m_vars[part] != NULL_TREE); | |
3578 | obj = m_vars[part]; | |
3579 | lhs = build_fold_addr_expr (obj); | |
3580 | } | |
3581 | else | |
3582 | { | |
3583 | lhs = build_fold_addr_expr (obj); | |
3584 | lhs = force_gimple_operand_gsi (&m_gsi, lhs, true, | |
3585 | NULL_TREE, true, GSI_SAME_STMT); | |
3586 | } | |
3587 | tree sitype = lang_hooks.types.type_for_mode (SImode, 0); | |
3588 | gimple *g; | |
3589 | switch (rhs_code) | |
3590 | { | |
3591 | case MULT_EXPR: | |
3592 | g = gimple_build_call_internal (IFN_MULBITINT, 6, | |
3593 | lhs, build_int_cst (sitype, prec), | |
3594 | rhs1, build_int_cst (sitype, prec1), | |
3595 | rhs2, build_int_cst (sitype, prec2)); | |
3596 | insert_before (g); | |
3597 | break; | |
3598 | case TRUNC_DIV_EXPR: | |
3599 | g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, | |
3600 | lhs, build_int_cst (sitype, prec), | |
3601 | null_pointer_node, | |
3602 | build_int_cst (sitype, 0), | |
3603 | rhs1, build_int_cst (sitype, prec1), | |
3604 | rhs2, build_int_cst (sitype, prec2)); | |
3605 | if (!stmt_ends_bb_p (stmt)) | |
3606 | gimple_call_set_nothrow (as_a <gcall *> (g), true); | |
3607 | insert_before (g); | |
3608 | break; | |
3609 | case TRUNC_MOD_EXPR: | |
3610 | g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node, | |
3611 | build_int_cst (sitype, 0), | |
3612 | lhs, build_int_cst (sitype, prec), | |
3613 | rhs1, build_int_cst (sitype, prec1), | |
3614 | rhs2, build_int_cst (sitype, prec2)); | |
3615 | if (!stmt_ends_bb_p (stmt)) | |
3616 | gimple_call_set_nothrow (as_a <gcall *> (g), true); | |
3617 | insert_before (g); | |
3618 | break; | |
3619 | default: | |
3620 | gcc_unreachable (); | |
3621 | } | |
3622 | if (stmt_ends_bb_p (stmt)) | |
3623 | { | |
3624 | maybe_duplicate_eh_stmt (g, stmt); | |
3625 | edge e1; | |
3626 | edge_iterator ei; | |
3627 | basic_block bb = gimple_bb (stmt); | |
3628 | ||
3629 | FOR_EACH_EDGE (e1, ei, bb->succs) | |
3630 | if (e1->flags & EDGE_EH) | |
3631 | break; | |
3632 | if (e1) | |
3633 | { | |
3634 | edge e2 = split_block (gsi_bb (m_gsi), g); | |
3635 | m_gsi = gsi_after_labels (e2->dest); | |
6c124873 | 3636 | add_eh_edge (e2->src, e1); |
a9d6c7fb JJ |
3637 | } |
3638 | } | |
3639 | } | |
3640 | ||
3641 | /* Lower large/huge _BitInt conversion to/from floating point. */ | |
3642 | ||
3643 | void | |
3644 | bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt) | |
3645 | { | |
3646 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
3647 | tree lhs = gimple_assign_lhs (stmt); | |
3648 | tree_code rhs_code = gimple_assign_rhs_code (stmt); | |
3649 | tree sitype = lang_hooks.types.type_for_mode (SImode, 0); | |
3650 | gimple *g; | |
3651 | if (rhs_code == FIX_TRUNC_EXPR) | |
3652 | { | |
3653 | int prec = TYPE_PRECISION (TREE_TYPE (lhs)); | |
3654 | if (!TYPE_UNSIGNED (TREE_TYPE (lhs))) | |
3655 | prec = -prec; | |
3656 | if (obj == NULL_TREE) | |
3657 | { | |
3658 | int part = var_to_partition (m_map, lhs); | |
3659 | gcc_assert (m_vars[part] != NULL_TREE); | |
3660 | obj = m_vars[part]; | |
3661 | lhs = build_fold_addr_expr (obj); | |
3662 | } | |
3663 | else | |
3664 | { | |
3665 | lhs = build_fold_addr_expr (obj); | |
3666 | lhs = force_gimple_operand_gsi (&m_gsi, lhs, true, | |
3667 | NULL_TREE, true, GSI_SAME_STMT); | |
3668 | } | |
3669 | scalar_mode from_mode | |
3670 | = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1))); | |
3671 | #ifdef HAVE_SFmode | |
3672 | /* IEEE single is a full superset of both IEEE half and | |
3673 | bfloat formats, convert to float first and then to _BitInt | |
3674 | to avoid the need of another 2 library routines. */ | |
3675 | if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format | |
3676 | || REAL_MODE_FORMAT (from_mode) == &ieee_half_format) | |
3677 | && REAL_MODE_FORMAT (SFmode) == &ieee_single_format) | |
3678 | { | |
3679 | tree type = lang_hooks.types.type_for_mode (SFmode, 0); | |
3680 | if (type) | |
3681 | rhs1 = add_cast (type, rhs1); | |
3682 | } | |
3683 | #endif | |
3684 | g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3, | |
3685 | lhs, build_int_cst (sitype, prec), | |
3686 | rhs1); | |
3687 | insert_before (g); | |
3688 | } | |
3689 | else | |
3690 | { | |
3691 | int prec; | |
3692 | rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec); | |
3693 | g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2, | |
3694 | rhs1, build_int_cst (sitype, prec)); | |
3695 | gimple_call_set_lhs (g, lhs); | |
3696 | if (!stmt_ends_bb_p (stmt)) | |
3697 | gimple_call_set_nothrow (as_a <gcall *> (g), true); | |
3698 | gsi_replace (&m_gsi, g, true); | |
3699 | } | |
3700 | } | |
3701 | ||
3702 | /* Helper method for lower_addsub_overflow and lower_mul_overflow. | |
3703 | If check_zero is true, caller wants to check if all bits in [start, end) | |
3704 | are zero, otherwise if bits in [start, end) are either all zero or | |
3705 | all ones. L is the limb with index LIMB, START and END are measured | |
3706 | in bits. */ | |
3707 | ||
3708 | tree | |
3709 | bitint_large_huge::arith_overflow_extract_bits (unsigned int start, | |
3710 | unsigned int end, tree l, | |
3711 | unsigned int limb, | |
3712 | bool check_zero) | |
3713 | { | |
3714 | unsigned startlimb = start / limb_prec; | |
3715 | unsigned endlimb = (end - 1) / limb_prec; | |
3716 | gimple *g; | |
3717 | ||
3718 | if ((start % limb_prec) == 0 && (end % limb_prec) == 0) | |
3719 | return l; | |
3720 | if (startlimb == endlimb && limb == startlimb) | |
3721 | { | |
3722 | if (check_zero) | |
3723 | { | |
3724 | wide_int w = wi::shifted_mask (start % limb_prec, | |
3725 | end - start, false, limb_prec); | |
3726 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3727 | BIT_AND_EXPR, l, | |
3728 | wide_int_to_tree (m_limb_type, w)); | |
3729 | insert_before (g); | |
3730 | return gimple_assign_lhs (g); | |
3731 | } | |
3732 | unsigned int shift = start % limb_prec; | |
3733 | if ((end % limb_prec) != 0) | |
3734 | { | |
3735 | unsigned int lshift = (-end) % limb_prec; | |
3736 | shift += lshift; | |
3737 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3738 | LSHIFT_EXPR, l, | |
3739 | build_int_cst (unsigned_type_node, | |
3740 | lshift)); | |
3741 | insert_before (g); | |
3742 | l = gimple_assign_lhs (g); | |
3743 | } | |
3744 | l = add_cast (signed_type_for (m_limb_type), l); | |
3745 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)), | |
3746 | RSHIFT_EXPR, l, | |
3747 | build_int_cst (unsigned_type_node, shift)); | |
3748 | insert_before (g); | |
3749 | return add_cast (m_limb_type, gimple_assign_lhs (g)); | |
3750 | } | |
3751 | else if (limb == startlimb) | |
3752 | { | |
3753 | if ((start % limb_prec) == 0) | |
3754 | return l; | |
3755 | if (!check_zero) | |
3756 | l = add_cast (signed_type_for (m_limb_type), l); | |
3757 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)), | |
3758 | RSHIFT_EXPR, l, | |
3759 | build_int_cst (unsigned_type_node, | |
3760 | start % limb_prec)); | |
3761 | insert_before (g); | |
3762 | l = gimple_assign_lhs (g); | |
3763 | if (!check_zero) | |
3764 | l = add_cast (m_limb_type, l); | |
3765 | return l; | |
3766 | } | |
3767 | else if (limb == endlimb) | |
3768 | { | |
3769 | if ((end % limb_prec) == 0) | |
3770 | return l; | |
3771 | if (check_zero) | |
3772 | { | |
3773 | wide_int w = wi::mask (end % limb_prec, false, limb_prec); | |
3774 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3775 | BIT_AND_EXPR, l, | |
3776 | wide_int_to_tree (m_limb_type, w)); | |
3777 | insert_before (g); | |
3778 | return gimple_assign_lhs (g); | |
3779 | } | |
3780 | unsigned int shift = (-end) % limb_prec; | |
3781 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
3782 | LSHIFT_EXPR, l, | |
3783 | build_int_cst (unsigned_type_node, shift)); | |
3784 | insert_before (g); | |
3785 | l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g)); | |
3786 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)), | |
3787 | RSHIFT_EXPR, l, | |
3788 | build_int_cst (unsigned_type_node, shift)); | |
3789 | insert_before (g); | |
3790 | return add_cast (m_limb_type, gimple_assign_lhs (g)); | |
3791 | } | |
3792 | return l; | |
3793 | } | |
3794 | ||
3795 | /* Helper method for lower_addsub_overflow and lower_mul_overflow. Store | |
3796 | result including overflow flag into the right locations. */ | |
3797 | ||
3798 | void | |
3799 | bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type, | |
3800 | tree ovf, tree lhs, tree orig_obj, | |
3801 | gimple *stmt, tree_code code) | |
3802 | { | |
3803 | gimple *g; | |
3804 | ||
3805 | if (obj == NULL_TREE | |
3806 | && (TREE_CODE (type) != BITINT_TYPE | |
3807 | || bitint_precision_kind (type) < bitint_prec_large)) | |
3808 | { | |
3809 | /* Add support for 3 or more limbs filled in from normal integral | |
3810 | type if this assert fails. If no target chooses limb mode smaller | |
3811 | than half of largest supported normal integral type, this will not | |
3812 | be needed. */ | |
3813 | gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec); | |
3814 | tree lhs_type = type; | |
3815 | if (TREE_CODE (type) == BITINT_TYPE | |
3816 | && bitint_precision_kind (type) == bitint_prec_middle) | |
3817 | lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type), | |
3818 | TYPE_UNSIGNED (type)); | |
3819 | tree r1 = limb_access (NULL_TREE, var, size_int (0), true); | |
3820 | g = gimple_build_assign (make_ssa_name (m_limb_type), r1); | |
3821 | insert_before (g); | |
3822 | r1 = gimple_assign_lhs (g); | |
3823 | if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1))) | |
3824 | r1 = add_cast (lhs_type, r1); | |
3825 | if (TYPE_PRECISION (lhs_type) > limb_prec) | |
3826 | { | |
3827 | tree r2 = limb_access (NULL_TREE, var, size_int (1), true); | |
3828 | g = gimple_build_assign (make_ssa_name (m_limb_type), r2); | |
3829 | insert_before (g); | |
3830 | r2 = gimple_assign_lhs (g); | |
3831 | r2 = add_cast (lhs_type, r2); | |
3832 | g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2, | |
3833 | build_int_cst (unsigned_type_node, | |
3834 | limb_prec)); | |
3835 | insert_before (g); | |
3836 | g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1, | |
3837 | gimple_assign_lhs (g)); | |
3838 | insert_before (g); | |
3839 | r1 = gimple_assign_lhs (g); | |
3840 | } | |
3841 | if (lhs_type != type) | |
3842 | r1 = add_cast (type, r1); | |
3843 | ovf = add_cast (lhs_type, ovf); | |
3844 | if (lhs_type != type) | |
3845 | ovf = add_cast (type, ovf); | |
3846 | g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf); | |
3847 | m_gsi = gsi_for_stmt (stmt); | |
3848 | gsi_replace (&m_gsi, g, true); | |
3849 | } | |
3850 | else | |
3851 | { | |
3852 | unsigned HOST_WIDE_INT nelts = 0; | |
3853 | tree atype = NULL_TREE; | |
3854 | if (obj) | |
3855 | { | |
3856 | nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec; | |
3857 | if (orig_obj == NULL_TREE) | |
3858 | nelts >>= 1; | |
3859 | atype = build_array_type_nelts (m_limb_type, nelts); | |
3860 | } | |
3861 | if (var && obj) | |
3862 | { | |
3863 | tree v1, v2; | |
3864 | tree zero; | |
3865 | if (orig_obj == NULL_TREE) | |
3866 | { | |
3867 | zero = build_zero_cst (build_pointer_type (TREE_TYPE (obj))); | |
3868 | v1 = build2 (MEM_REF, atype, | |
3869 | build_fold_addr_expr (unshare_expr (obj)), zero); | |
3870 | } | |
3871 | else if (!useless_type_conversion_p (atype, TREE_TYPE (obj))) | |
3872 | v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj)); | |
3873 | else | |
3874 | v1 = unshare_expr (obj); | |
3875 | zero = build_zero_cst (build_pointer_type (TREE_TYPE (var))); | |
3876 | v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), zero); | |
3877 | g = gimple_build_assign (v1, v2); | |
3878 | insert_before (g); | |
3879 | } | |
3880 | if (orig_obj == NULL_TREE && obj) | |
3881 | { | |
3882 | ovf = add_cast (m_limb_type, ovf); | |
3883 | tree l = limb_access (NULL_TREE, obj, size_int (nelts), true); | |
3884 | g = gimple_build_assign (l, ovf); | |
3885 | insert_before (g); | |
3886 | if (nelts > 1) | |
3887 | { | |
3888 | atype = build_array_type_nelts (m_limb_type, nelts - 1); | |
3889 | tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)), | |
3890 | (nelts + 1) * m_limb_size); | |
3891 | tree v1 = build2 (MEM_REF, atype, | |
3892 | build_fold_addr_expr (unshare_expr (obj)), | |
3893 | off); | |
3894 | g = gimple_build_assign (v1, build_zero_cst (atype)); | |
3895 | insert_before (g); | |
3896 | } | |
3897 | } | |
3898 | else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE) | |
3899 | { | |
3900 | imm_use_iterator ui; | |
3901 | use_operand_p use_p; | |
3902 | FOR_EACH_IMM_USE_FAST (use_p, ui, lhs) | |
3903 | { | |
3904 | g = USE_STMT (use_p); | |
3905 | if (!is_gimple_assign (g) | |
3906 | || gimple_assign_rhs_code (g) != IMAGPART_EXPR) | |
3907 | continue; | |
3908 | tree lhs2 = gimple_assign_lhs (g); | |
3909 | gimple *use_stmt; | |
3910 | single_imm_use (lhs2, &use_p, &use_stmt); | |
3911 | lhs2 = gimple_assign_lhs (use_stmt); | |
3912 | gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt); | |
3913 | if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf))) | |
3914 | g = gimple_build_assign (lhs2, ovf); | |
3915 | else | |
3916 | g = gimple_build_assign (lhs2, NOP_EXPR, ovf); | |
3917 | gsi_replace (&gsi, g, true); | |
0ef93c86 JJ |
3918 | if (gsi_stmt (m_gsi) == use_stmt) |
3919 | m_gsi = gsi_for_stmt (g); | |
a9d6c7fb JJ |
3920 | break; |
3921 | } | |
3922 | } | |
3923 | else if (ovf != boolean_false_node) | |
3924 | { | |
3925 | g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node, | |
3926 | NULL_TREE, NULL_TREE); | |
3927 | edge edge_true, edge_false; | |
3928 | if_then (g, profile_probability::very_unlikely (), | |
3929 | edge_true, edge_false); | |
3930 | tree zero = build_zero_cst (TREE_TYPE (lhs)); | |
3931 | tree fn = ubsan_build_overflow_builtin (code, m_loc, | |
3932 | TREE_TYPE (lhs), | |
3933 | zero, zero, NULL); | |
3934 | force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE, | |
3935 | true, GSI_SAME_STMT); | |
3936 | m_gsi = gsi_after_labels (edge_true->dest); | |
3937 | } | |
3938 | } | |
3939 | if (var) | |
3940 | { | |
de072b52 | 3941 | tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END); |
a9d6c7fb JJ |
3942 | g = gimple_build_assign (var, clobber); |
3943 | gsi_insert_after (&m_gsi, g, GSI_SAME_STMT); | |
3944 | } | |
3945 | } | |
3946 | ||
3947 | /* Helper function for lower_addsub_overflow and lower_mul_overflow. | |
3948 | Given precisions of result TYPE (PREC), argument 0 precision PREC0, | |
3949 | argument 1 precision PREC1 and minimum precision for the result | |
3950 | PREC2, compute *START, *END, *CHECK_ZERO and return OVF. */ | |
3951 | ||
3952 | static tree | |
3953 | arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1, | |
3954 | int prec2, unsigned *start, unsigned *end, bool *check_zero) | |
3955 | { | |
3956 | *start = 0; | |
3957 | *end = 0; | |
3958 | *check_zero = true; | |
3959 | /* Ignore this special rule for subtraction, even if both | |
3960 | prec0 >= 0 and prec1 >= 0, their subtraction can be negative | |
3961 | in infinite precision. */ | |
3962 | if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0) | |
3963 | { | |
3964 | /* Result in [0, prec2) is unsigned, if prec > prec2, | |
3965 | all bits above it will be zero. */ | |
3966 | if ((prec - !TYPE_UNSIGNED (type)) >= prec2) | |
3967 | return boolean_false_node; | |
3968 | else | |
3969 | { | |
3970 | /* ovf if any of bits in [start, end) is non-zero. */ | |
3971 | *start = prec - !TYPE_UNSIGNED (type); | |
3972 | *end = prec2; | |
3973 | } | |
3974 | } | |
3975 | else if (TYPE_UNSIGNED (type)) | |
3976 | { | |
3977 | /* If result in [0, prec2) is signed and if prec > prec2, | |
3978 | all bits above it will be sign bit copies. */ | |
3979 | if (prec >= prec2) | |
3980 | { | |
3981 | /* ovf if bit prec - 1 is non-zero. */ | |
3982 | *start = prec - 1; | |
3983 | *end = prec; | |
3984 | } | |
3985 | else | |
3986 | { | |
3987 | /* ovf if any of bits in [start, end) is non-zero. */ | |
3988 | *start = prec; | |
3989 | *end = prec2; | |
3990 | } | |
3991 | } | |
3992 | else if (prec >= prec2) | |
3993 | return boolean_false_node; | |
3994 | else | |
3995 | { | |
3996 | /* ovf if [start, end) bits aren't all zeros or all ones. */ | |
3997 | *start = prec - 1; | |
3998 | *end = prec2; | |
3999 | *check_zero = false; | |
4000 | } | |
4001 | return NULL_TREE; | |
4002 | } | |
4003 | ||
4004 | /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt | |
4005 | argument or return type _Complex large/huge _BitInt. */ | |
4006 | ||
4007 | void | |
4008 | bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt) | |
4009 | { | |
4010 | tree arg0 = gimple_call_arg (stmt, 0); | |
4011 | tree arg1 = gimple_call_arg (stmt, 1); | |
4012 | tree lhs = gimple_call_lhs (stmt); | |
4013 | gimple *g; | |
4014 | ||
4015 | if (!lhs) | |
4016 | { | |
4017 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
4018 | gsi_remove (&gsi, true); | |
4019 | return; | |
4020 | } | |
4021 | gimple *final_stmt = gsi_stmt (m_gsi); | |
4022 | tree type = TREE_TYPE (lhs); | |
4023 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
4024 | type = TREE_TYPE (type); | |
4025 | int prec = TYPE_PRECISION (type); | |
4026 | int prec0 = range_to_prec (arg0, stmt); | |
4027 | int prec1 = range_to_prec (arg1, stmt); | |
e3be66df JJ |
4028 | /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is |
4029 | the be minimum unsigned precision of any possible operation's | |
4030 | result, otherwise it is minimum signed precision. | |
4031 | Some examples: | |
4032 | If PREC0 or PREC1 is 8, it means that argument is [0, 0xff], | |
4033 | if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff], | |
4034 | if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f], | |
4035 | if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff]. | |
4036 | PREC0 CODE PREC1 RESULT PREC2 SIGNED vs. UNSIGNED | |
4037 | 8 + 8 [0, 0x1fe] 9 UNSIGNED | |
4038 | 8 + 10 [0, 0x4fe] 11 UNSIGNED | |
4039 | -8 + -8 [-0x100, 0xfe] 9 SIGNED | |
4040 | -8 + -10 [-0x280, 0x27e] 11 SIGNED | |
4041 | 8 + -8 [-0x80, 0x17e] 10 SIGNED | |
4042 | 8 + -10 [-0x200, 0x2fe] 11 SIGNED | |
4043 | 10 + -8 [-0x80, 0x47e] 12 SIGNED | |
4044 | 8 - 8 [-0xff, 0xff] 9 SIGNED | |
4045 | 8 - 10 [-0x3ff, 0xff] 11 SIGNED | |
4046 | 10 - 8 [-0xff, 0x3ff] 11 SIGNED | |
4047 | -8 - -8 [-0xff, 0xff] 9 SIGNED | |
4048 | -8 - -10 [-0x27f, 0x27f] 11 SIGNED | |
4049 | -10 - -8 [-0x27f, 0x27f] 11 SIGNED | |
4050 | 8 - -8 [-0x7f, 0x17f] 10 SIGNED | |
4051 | 8 - -10 [-0x1ff, 0x2ff] 11 SIGNED | |
4052 | 10 - -8 [-0x7f, 0x47f] 12 SIGNED | |
4053 | -8 - 8 [-0x17f, 0x7f] 10 SIGNED | |
4054 | -8 - 10 [-0x47f, 0x7f] 12 SIGNED | |
4055 | -10 - 8 [-0x2ff, 0x1ff] 11 SIGNED */ | |
4056 | int prec2 = MAX (prec0 < 0 ? -prec0 : prec0, | |
4057 | prec1 < 0 ? -prec1 : prec1); | |
4058 | /* If operands are either both signed or both unsigned, | |
4059 | we need just one additional bit. */ | |
4060 | prec2 = (((prec0 < 0) == (prec1 < 0) | |
4061 | /* If one operand is signed and one unsigned and | |
4062 | the signed one has larger precision, we need | |
4063 | just one extra bit, otherwise two. */ | |
4064 | || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1) | |
4065 | : (prec2 == -prec1 && prec2 != prec0))) | |
4066 | ? prec2 + 1 : prec2 + 2); | |
a9d6c7fb JJ |
4067 | int prec3 = MAX (prec0 < 0 ? -prec0 : prec0, |
4068 | prec1 < 0 ? -prec1 : prec1); | |
4069 | prec3 = MAX (prec3, prec); | |
4070 | tree var = NULL_TREE; | |
4071 | tree orig_obj = obj; | |
4072 | if (obj == NULL_TREE | |
4073 | && TREE_CODE (type) == BITINT_TYPE | |
4074 | && bitint_precision_kind (type) >= bitint_prec_large | |
4075 | && m_names | |
4076 | && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs))) | |
4077 | { | |
4078 | int part = var_to_partition (m_map, lhs); | |
4079 | gcc_assert (m_vars[part] != NULL_TREE); | |
4080 | obj = m_vars[part]; | |
4081 | if (TREE_TYPE (lhs) == type) | |
4082 | orig_obj = obj; | |
4083 | } | |
4084 | if (TREE_CODE (type) != BITINT_TYPE | |
4085 | || bitint_precision_kind (type) < bitint_prec_large) | |
4086 | { | |
4087 | unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec); | |
4088 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
4089 | var = create_tmp_var (atype); | |
4090 | } | |
4091 | ||
4092 | enum tree_code code; | |
4093 | switch (gimple_call_internal_fn (stmt)) | |
4094 | { | |
4095 | case IFN_ADD_OVERFLOW: | |
4096 | case IFN_UBSAN_CHECK_ADD: | |
4097 | code = PLUS_EXPR; | |
4098 | break; | |
4099 | case IFN_SUB_OVERFLOW: | |
4100 | case IFN_UBSAN_CHECK_SUB: | |
4101 | code = MINUS_EXPR; | |
4102 | break; | |
4103 | default: | |
4104 | gcc_unreachable (); | |
4105 | } | |
4106 | unsigned start, end; | |
4107 | bool check_zero; | |
4108 | tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2, | |
4109 | &start, &end, &check_zero); | |
4110 | ||
4111 | unsigned startlimb, endlimb; | |
4112 | if (ovf) | |
4113 | { | |
4114 | startlimb = ~0U; | |
4115 | endlimb = ~0U; | |
4116 | } | |
4117 | else | |
4118 | { | |
4119 | startlimb = start / limb_prec; | |
4120 | endlimb = (end - 1) / limb_prec; | |
4121 | } | |
4122 | ||
4123 | int prec4 = ovf != NULL_TREE ? prec : prec3; | |
4124 | bitint_prec_kind kind = bitint_precision_kind (prec4); | |
4125 | unsigned cnt, rem = 0, fin = 0; | |
4126 | tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE; | |
4127 | bool last_ovf = (ovf == NULL_TREE | |
4128 | && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec)); | |
4129 | if (kind != bitint_prec_huge) | |
4130 | cnt = CEIL (prec4, limb_prec) + last_ovf; | |
4131 | else | |
4132 | { | |
4133 | rem = (prec4 % (2 * limb_prec)); | |
4134 | fin = (prec4 - rem) / limb_prec; | |
4135 | cnt = 2 + CEIL (rem, limb_prec) + last_ovf; | |
4136 | idx = idx_first = create_loop (size_zero_node, &idx_next); | |
4137 | } | |
4138 | ||
4139 | if (kind == bitint_prec_huge) | |
4140 | m_upwards_2limb = fin; | |
4141 | m_upwards = true; | |
4142 | ||
4143 | tree type0 = TREE_TYPE (arg0); | |
4144 | tree type1 = TREE_TYPE (arg1); | |
eef6aea3 JJ |
4145 | int prec5 = prec3; |
4146 | if (bitint_precision_kind (prec5) < bitint_prec_large) | |
4147 | prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1)); | |
4148 | if (TYPE_PRECISION (type0) < prec5) | |
a9d6c7fb | 4149 | { |
eef6aea3 | 4150 | type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0)); |
a9d6c7fb JJ |
4151 | if (TREE_CODE (arg0) == INTEGER_CST) |
4152 | arg0 = fold_convert (type0, arg0); | |
4153 | } | |
eef6aea3 | 4154 | if (TYPE_PRECISION (type1) < prec5) |
a9d6c7fb | 4155 | { |
eef6aea3 | 4156 | type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1)); |
a9d6c7fb JJ |
4157 | if (TREE_CODE (arg1) == INTEGER_CST) |
4158 | arg1 = fold_convert (type1, arg1); | |
4159 | } | |
4160 | unsigned int data_cnt = 0; | |
4161 | tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE; | |
4162 | tree cmp = build_zero_cst (m_limb_type); | |
4163 | unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec); | |
4164 | tree ovf_out = NULL_TREE, cmp_out = NULL_TREE; | |
4165 | for (unsigned i = 0; i < cnt; i++) | |
4166 | { | |
4167 | m_data_cnt = 0; | |
4168 | tree rhs1, rhs2; | |
4169 | if (kind != bitint_prec_huge) | |
4170 | idx = size_int (i); | |
4171 | else if (i >= 2) | |
d10c0dce | 4172 | idx = size_int (fin + i - 2); |
a9d6c7fb JJ |
4173 | if (!last_ovf || i < cnt - 1) |
4174 | { | |
4175 | if (type0 != TREE_TYPE (arg0)) | |
4176 | rhs1 = handle_cast (type0, arg0, idx); | |
4177 | else | |
4178 | rhs1 = handle_operand (arg0, idx); | |
4179 | if (type1 != TREE_TYPE (arg1)) | |
4180 | rhs2 = handle_cast (type1, arg1, idx); | |
4181 | else | |
4182 | rhs2 = handle_operand (arg1, idx); | |
4183 | if (i == 0) | |
4184 | data_cnt = m_data_cnt; | |
4185 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1))) | |
4186 | rhs1 = add_cast (m_limb_type, rhs1); | |
4187 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2))) | |
4188 | rhs2 = add_cast (m_limb_type, rhs2); | |
4189 | last_rhs1 = rhs1; | |
4190 | last_rhs2 = rhs2; | |
4191 | } | |
4192 | else | |
4193 | { | |
4194 | m_data_cnt = data_cnt; | |
4195 | if (TYPE_UNSIGNED (type0)) | |
4196 | rhs1 = build_zero_cst (m_limb_type); | |
4197 | else | |
4198 | { | |
4199 | rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1); | |
4200 | if (TREE_CODE (rhs1) == INTEGER_CST) | |
4201 | rhs1 = build_int_cst (m_limb_type, | |
4202 | tree_int_cst_sgn (rhs1) < 0 ? -1 : 0); | |
4203 | else | |
4204 | { | |
4205 | tree lpm1 = build_int_cst (unsigned_type_node, | |
4206 | limb_prec - 1); | |
4207 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)), | |
4208 | RSHIFT_EXPR, rhs1, lpm1); | |
4209 | insert_before (g); | |
4210 | rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g)); | |
4211 | } | |
4212 | } | |
4213 | if (TYPE_UNSIGNED (type1)) | |
4214 | rhs2 = build_zero_cst (m_limb_type); | |
4215 | else | |
4216 | { | |
4217 | rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2); | |
4218 | if (TREE_CODE (rhs2) == INTEGER_CST) | |
4219 | rhs2 = build_int_cst (m_limb_type, | |
4220 | tree_int_cst_sgn (rhs2) < 0 ? -1 : 0); | |
4221 | else | |
4222 | { | |
4223 | tree lpm1 = build_int_cst (unsigned_type_node, | |
4224 | limb_prec - 1); | |
4225 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)), | |
4226 | RSHIFT_EXPR, rhs2, lpm1); | |
4227 | insert_before (g); | |
4228 | rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g)); | |
4229 | } | |
4230 | } | |
4231 | } | |
4232 | tree rhs = handle_plus_minus (code, rhs1, rhs2, idx); | |
4233 | if (ovf != boolean_false_node) | |
4234 | { | |
4235 | if (tree_fits_uhwi_p (idx)) | |
4236 | { | |
4237 | unsigned limb = tree_to_uhwi (idx); | |
4238 | if (limb >= startlimb && limb <= endlimb) | |
4239 | { | |
4240 | tree l = arith_overflow_extract_bits (start, end, rhs, | |
4241 | limb, check_zero); | |
4242 | tree this_ovf = make_ssa_name (boolean_type_node); | |
4243 | if (ovf == NULL_TREE && !check_zero) | |
4244 | { | |
4245 | cmp = l; | |
4246 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
4247 | PLUS_EXPR, l, | |
4248 | build_int_cst (m_limb_type, 1)); | |
4249 | insert_before (g); | |
4250 | g = gimple_build_assign (this_ovf, GT_EXPR, | |
4251 | gimple_assign_lhs (g), | |
4252 | build_int_cst (m_limb_type, 1)); | |
4253 | } | |
4254 | else | |
4255 | g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp); | |
4256 | insert_before (g); | |
4257 | if (ovf == NULL_TREE) | |
4258 | ovf = this_ovf; | |
4259 | else | |
4260 | { | |
4261 | tree b = make_ssa_name (boolean_type_node); | |
4262 | g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf); | |
4263 | insert_before (g); | |
4264 | ovf = b; | |
4265 | } | |
4266 | } | |
4267 | } | |
4268 | else if (startlimb < fin) | |
4269 | { | |
4270 | if (m_first && startlimb + 2 < fin) | |
4271 | { | |
4272 | tree data_out; | |
4273 | ovf = prepare_data_in_out (boolean_false_node, idx, &data_out); | |
4274 | ovf_out = m_data.pop (); | |
4275 | m_data.pop (); | |
4276 | if (!check_zero) | |
4277 | { | |
4278 | cmp = prepare_data_in_out (cmp, idx, &data_out); | |
4279 | cmp_out = m_data.pop (); | |
4280 | m_data.pop (); | |
4281 | } | |
4282 | } | |
4283 | if (i != 0 || startlimb != fin - 1) | |
4284 | { | |
4285 | tree_code cmp_code; | |
4286 | bool single_comparison | |
4287 | = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1)); | |
4288 | if (!single_comparison) | |
4289 | { | |
4290 | cmp_code = GE_EXPR; | |
4291 | if (!check_zero && (start % limb_prec) == 0) | |
4292 | single_comparison = true; | |
4293 | } | |
4294 | else if ((startlimb & 1) == (i & 1)) | |
4295 | cmp_code = EQ_EXPR; | |
4296 | else | |
4297 | cmp_code = GT_EXPR; | |
4298 | g = gimple_build_cond (cmp_code, idx, size_int (startlimb), | |
4299 | NULL_TREE, NULL_TREE); | |
4300 | edge edge_true_true, edge_true_false, edge_false; | |
4301 | gimple *g2 = NULL; | |
4302 | if (!single_comparison) | |
36433265 | 4303 | g2 = gimple_build_cond (NE_EXPR, idx, |
a9d6c7fb JJ |
4304 | size_int (startlimb), NULL_TREE, |
4305 | NULL_TREE); | |
4306 | if_then_if_then_else (g, g2, profile_probability::likely (), | |
36433265 | 4307 | profile_probability::likely (), |
a9d6c7fb JJ |
4308 | edge_true_true, edge_true_false, |
4309 | edge_false); | |
4310 | unsigned tidx = startlimb + (cmp_code == GT_EXPR); | |
4311 | tree l = arith_overflow_extract_bits (start, end, rhs, tidx, | |
4312 | check_zero); | |
4313 | tree this_ovf = make_ssa_name (boolean_type_node); | |
4314 | if (cmp_code != GT_EXPR && !check_zero) | |
4315 | { | |
4316 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
4317 | PLUS_EXPR, l, | |
4318 | build_int_cst (m_limb_type, 1)); | |
4319 | insert_before (g); | |
4320 | g = gimple_build_assign (this_ovf, GT_EXPR, | |
4321 | gimple_assign_lhs (g), | |
4322 | build_int_cst (m_limb_type, 1)); | |
4323 | } | |
4324 | else | |
4325 | g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp); | |
4326 | insert_before (g); | |
4327 | if (cmp_code == GT_EXPR) | |
4328 | { | |
4329 | tree t = make_ssa_name (boolean_type_node); | |
4330 | g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf); | |
4331 | insert_before (g); | |
4332 | this_ovf = t; | |
4333 | } | |
4334 | tree this_ovf2 = NULL_TREE; | |
4335 | if (!single_comparison) | |
4336 | { | |
4337 | m_gsi = gsi_after_labels (edge_true_true->src); | |
4338 | tree t = make_ssa_name (boolean_type_node); | |
4339 | g = gimple_build_assign (t, NE_EXPR, rhs, cmp); | |
4340 | insert_before (g); | |
4341 | this_ovf2 = make_ssa_name (boolean_type_node); | |
4342 | g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR, | |
4343 | ovf, t); | |
4344 | insert_before (g); | |
4345 | } | |
4346 | m_gsi = gsi_after_labels (edge_true_false->dest); | |
4347 | tree t; | |
4348 | if (i == 1 && ovf_out) | |
4349 | t = ovf_out; | |
4350 | else | |
4351 | t = make_ssa_name (boolean_type_node); | |
4352 | gphi *phi = create_phi_node (t, edge_true_false->dest); | |
4353 | add_phi_arg (phi, this_ovf, edge_true_false, | |
4354 | UNKNOWN_LOCATION); | |
4355 | add_phi_arg (phi, ovf ? ovf | |
4356 | : boolean_false_node, edge_false, | |
4357 | UNKNOWN_LOCATION); | |
4358 | if (edge_true_true) | |
4359 | add_phi_arg (phi, this_ovf2, edge_true_true, | |
4360 | UNKNOWN_LOCATION); | |
4361 | ovf = t; | |
4362 | if (!check_zero && cmp_code != GT_EXPR) | |
4363 | { | |
4364 | t = cmp_out ? cmp_out : make_ssa_name (m_limb_type); | |
4365 | phi = create_phi_node (t, edge_true_false->dest); | |
4366 | add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION); | |
4367 | add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION); | |
4368 | if (edge_true_true) | |
4369 | add_phi_arg (phi, cmp, edge_true_true, | |
4370 | UNKNOWN_LOCATION); | |
4371 | cmp = t; | |
4372 | } | |
4373 | } | |
4374 | } | |
4375 | } | |
4376 | ||
4377 | if (var || obj) | |
4378 | { | |
4379 | if (tree_fits_uhwi_p (idx) && tree_to_uhwi (idx) >= prec_limbs) | |
4380 | ; | |
4381 | else if (!tree_fits_uhwi_p (idx) | |
4382 | && (unsigned) prec < (fin - (i == 0)) * limb_prec) | |
4383 | { | |
4384 | bool single_comparison | |
4385 | = (((unsigned) prec % limb_prec) == 0 | |
4386 | || prec_limbs + 1 >= fin | |
4387 | || (prec_limbs & 1) == (i & 1)); | |
4388 | g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1), | |
4389 | NULL_TREE, NULL_TREE); | |
4390 | gimple *g2 = NULL; | |
4391 | if (!single_comparison) | |
be1f2bc4 | 4392 | g2 = gimple_build_cond (EQ_EXPR, idx, |
a9d6c7fb JJ |
4393 | size_int (prec_limbs - 1), |
4394 | NULL_TREE, NULL_TREE); | |
4395 | edge edge_true_true, edge_true_false, edge_false; | |
4396 | if_then_if_then_else (g, g2, profile_probability::likely (), | |
be1f2bc4 | 4397 | profile_probability::unlikely (), |
a9d6c7fb JJ |
4398 | edge_true_true, edge_true_false, |
4399 | edge_false); | |
4400 | tree l = limb_access (type, var ? var : obj, idx, true); | |
4401 | g = gimple_build_assign (l, rhs); | |
4402 | insert_before (g); | |
4403 | if (!single_comparison) | |
4404 | { | |
4405 | m_gsi = gsi_after_labels (edge_true_true->src); | |
be1f2bc4 JJ |
4406 | tree plm1idx = size_int (prec_limbs - 1); |
4407 | tree plm1type = limb_access_type (type, plm1idx); | |
4408 | l = limb_access (type, var ? var : obj, plm1idx, true); | |
4409 | if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs))) | |
4410 | rhs = add_cast (plm1type, rhs); | |
a9d6c7fb JJ |
4411 | if (!useless_type_conversion_p (TREE_TYPE (l), |
4412 | TREE_TYPE (rhs))) | |
4413 | rhs = add_cast (TREE_TYPE (l), rhs); | |
4414 | g = gimple_build_assign (l, rhs); | |
4415 | insert_before (g); | |
4416 | } | |
4417 | m_gsi = gsi_after_labels (edge_true_false->dest); | |
4418 | } | |
4419 | else | |
4420 | { | |
4421 | tree l = limb_access (type, var ? var : obj, idx, true); | |
4422 | if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs))) | |
4423 | rhs = add_cast (TREE_TYPE (l), rhs); | |
4424 | g = gimple_build_assign (l, rhs); | |
4425 | insert_before (g); | |
4426 | } | |
4427 | } | |
4428 | m_first = false; | |
4429 | if (kind == bitint_prec_huge && i <= 1) | |
4430 | { | |
4431 | if (i == 0) | |
4432 | { | |
4433 | idx = make_ssa_name (sizetype); | |
4434 | g = gimple_build_assign (idx, PLUS_EXPR, idx_first, | |
4435 | size_one_node); | |
4436 | insert_before (g); | |
4437 | } | |
4438 | else | |
4439 | { | |
4440 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first, | |
4441 | size_int (2)); | |
4442 | insert_before (g); | |
4443 | g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin), | |
4444 | NULL_TREE, NULL_TREE); | |
4445 | insert_before (g); | |
4446 | m_gsi = gsi_for_stmt (final_stmt); | |
be9e8de6 | 4447 | m_bb = NULL; |
a9d6c7fb JJ |
4448 | } |
4449 | } | |
4450 | } | |
4451 | ||
4452 | finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt, code); | |
4453 | } | |
4454 | ||
4455 | /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt | |
4456 | argument or return type _Complex large/huge _BitInt. */ | |
4457 | ||
4458 | void | |
4459 | bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt) | |
4460 | { | |
4461 | tree arg0 = gimple_call_arg (stmt, 0); | |
4462 | tree arg1 = gimple_call_arg (stmt, 1); | |
4463 | tree lhs = gimple_call_lhs (stmt); | |
4464 | if (!lhs) | |
4465 | { | |
4466 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
4467 | gsi_remove (&gsi, true); | |
4468 | return; | |
4469 | } | |
4470 | gimple *final_stmt = gsi_stmt (m_gsi); | |
4471 | tree type = TREE_TYPE (lhs); | |
4472 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
4473 | type = TREE_TYPE (type); | |
4474 | int prec = TYPE_PRECISION (type), prec0, prec1; | |
4475 | arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0); | |
4476 | arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1); | |
4477 | int prec2 = ((prec0 < 0 ? -prec0 : prec0) | |
e3be66df JJ |
4478 | + (prec1 < 0 ? -prec1 : prec1)); |
4479 | if (prec0 == 1 || prec1 == 1) | |
4480 | --prec2; | |
a9d6c7fb JJ |
4481 | tree var = NULL_TREE; |
4482 | tree orig_obj = obj; | |
4483 | bool force_var = false; | |
4484 | if (obj == NULL_TREE | |
4485 | && TREE_CODE (type) == BITINT_TYPE | |
4486 | && bitint_precision_kind (type) >= bitint_prec_large | |
4487 | && m_names | |
4488 | && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs))) | |
4489 | { | |
4490 | int part = var_to_partition (m_map, lhs); | |
4491 | gcc_assert (m_vars[part] != NULL_TREE); | |
4492 | obj = m_vars[part]; | |
4493 | if (TREE_TYPE (lhs) == type) | |
4494 | orig_obj = obj; | |
4495 | } | |
4496 | else if (obj != NULL_TREE && DECL_P (obj)) | |
4497 | { | |
4498 | for (int i = 0; i < 2; ++i) | |
4499 | { | |
4500 | tree arg = i ? arg1 : arg0; | |
4501 | if (TREE_CODE (arg) == ADDR_EXPR) | |
4502 | arg = TREE_OPERAND (arg, 0); | |
4503 | if (get_base_address (arg) == obj) | |
4504 | { | |
4505 | force_var = true; | |
4506 | break; | |
4507 | } | |
4508 | } | |
4509 | } | |
4510 | if (obj == NULL_TREE | |
4511 | || force_var | |
4512 | || TREE_CODE (type) != BITINT_TYPE | |
4513 | || bitint_precision_kind (type) < bitint_prec_large | |
4514 | || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2))) | |
4515 | { | |
4516 | unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec); | |
4517 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
4518 | var = create_tmp_var (atype); | |
4519 | } | |
4520 | tree addr = build_fold_addr_expr (var ? var : obj); | |
4521 | addr = force_gimple_operand_gsi (&m_gsi, addr, true, | |
4522 | NULL_TREE, true, GSI_SAME_STMT); | |
4523 | tree sitype = lang_hooks.types.type_for_mode (SImode, 0); | |
4524 | gimple *g | |
4525 | = gimple_build_call_internal (IFN_MULBITINT, 6, | |
4526 | addr, build_int_cst (sitype, | |
4527 | MAX (prec2, prec)), | |
4528 | arg0, build_int_cst (sitype, prec0), | |
4529 | arg1, build_int_cst (sitype, prec1)); | |
4530 | insert_before (g); | |
4531 | ||
4532 | unsigned start, end; | |
4533 | bool check_zero; | |
4534 | tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2, | |
4535 | &start, &end, &check_zero); | |
4536 | if (ovf == NULL_TREE) | |
4537 | { | |
4538 | unsigned startlimb = start / limb_prec; | |
4539 | unsigned endlimb = (end - 1) / limb_prec; | |
4540 | unsigned cnt; | |
4541 | bool use_loop = false; | |
4542 | if (startlimb == endlimb) | |
4543 | cnt = 1; | |
4544 | else if (startlimb + 1 == endlimb) | |
4545 | cnt = 2; | |
4546 | else if ((end % limb_prec) == 0) | |
4547 | { | |
4548 | cnt = 2; | |
4549 | use_loop = true; | |
4550 | } | |
4551 | else | |
4552 | { | |
4553 | cnt = 3; | |
4554 | use_loop = startlimb + 2 < endlimb; | |
4555 | } | |
4556 | if (cnt == 1) | |
4557 | { | |
4558 | tree l = limb_access (NULL_TREE, var ? var : obj, | |
4559 | size_int (startlimb), true); | |
4560 | g = gimple_build_assign (make_ssa_name (m_limb_type), l); | |
4561 | insert_before (g); | |
4562 | l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g), | |
4563 | startlimb, check_zero); | |
4564 | ovf = make_ssa_name (boolean_type_node); | |
4565 | if (check_zero) | |
4566 | g = gimple_build_assign (ovf, NE_EXPR, l, | |
4567 | build_zero_cst (m_limb_type)); | |
4568 | else | |
4569 | { | |
4570 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
4571 | PLUS_EXPR, l, | |
4572 | build_int_cst (m_limb_type, 1)); | |
4573 | insert_before (g); | |
4574 | g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g), | |
4575 | build_int_cst (m_limb_type, 1)); | |
4576 | } | |
4577 | insert_before (g); | |
4578 | } | |
4579 | else | |
4580 | { | |
4581 | basic_block edge_bb = NULL; | |
4582 | gimple_stmt_iterator gsi = m_gsi; | |
4583 | gsi_prev (&gsi); | |
4584 | edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
4585 | edge_bb = e->src; | |
9a96a9e4 | 4586 | m_gsi = gsi_end_bb (edge_bb); |
a9d6c7fb JJ |
4587 | |
4588 | tree cmp = build_zero_cst (m_limb_type); | |
4589 | for (unsigned i = 0; i < cnt; i++) | |
4590 | { | |
4591 | tree idx, idx_next = NULL_TREE; | |
4592 | if (i == 0) | |
4593 | idx = size_int (startlimb); | |
4594 | else if (i == 2) | |
4595 | idx = size_int (endlimb); | |
4596 | else if (use_loop) | |
4597 | idx = create_loop (size_int (startlimb + 1), &idx_next); | |
4598 | else | |
4599 | idx = size_int (startlimb + 1); | |
4600 | tree l = limb_access (NULL_TREE, var ? var : obj, idx, true); | |
4601 | g = gimple_build_assign (make_ssa_name (m_limb_type), l); | |
4602 | insert_before (g); | |
4603 | l = gimple_assign_lhs (g); | |
4604 | if (i == 0 || i == 2) | |
4605 | l = arith_overflow_extract_bits (start, end, l, | |
4606 | tree_to_uhwi (idx), | |
4607 | check_zero); | |
4608 | if (i == 0 && !check_zero) | |
4609 | { | |
4610 | cmp = l; | |
4611 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
4612 | PLUS_EXPR, l, | |
4613 | build_int_cst (m_limb_type, 1)); | |
4614 | insert_before (g); | |
4615 | g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g), | |
4616 | build_int_cst (m_limb_type, 1), | |
4617 | NULL_TREE, NULL_TREE); | |
4618 | } | |
4619 | else | |
4620 | g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE); | |
4621 | insert_before (g); | |
4622 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
4623 | e1->flags = EDGE_FALSE_VALUE; | |
4624 | edge e2 = make_edge (e1->src, gimple_bb (final_stmt), | |
4625 | EDGE_TRUE_VALUE); | |
4626 | e1->probability = profile_probability::likely (); | |
4627 | e2->probability = e1->probability.invert (); | |
4628 | if (i == 0) | |
4629 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
4630 | m_gsi = gsi_after_labels (e1->dest); | |
4631 | if (i == 1 && use_loop) | |
4632 | { | |
4633 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, | |
4634 | size_one_node); | |
4635 | insert_before (g); | |
4636 | g = gimple_build_cond (NE_EXPR, idx_next, | |
853cbcb7 | 4637 | size_int (endlimb + (cnt == 2)), |
a9d6c7fb JJ |
4638 | NULL_TREE, NULL_TREE); |
4639 | insert_before (g); | |
4640 | edge true_edge, false_edge; | |
4641 | extract_true_false_edges_from_block (gsi_bb (m_gsi), | |
4642 | &true_edge, | |
4643 | &false_edge); | |
4644 | m_gsi = gsi_after_labels (false_edge->dest); | |
be9e8de6 | 4645 | m_bb = NULL; |
a9d6c7fb JJ |
4646 | } |
4647 | } | |
4648 | ||
4649 | ovf = make_ssa_name (boolean_type_node); | |
4650 | basic_block bb = gimple_bb (final_stmt); | |
4651 | gphi *phi = create_phi_node (ovf, bb); | |
4652 | edge e1 = find_edge (gsi_bb (m_gsi), bb); | |
4653 | edge_iterator ei; | |
4654 | FOR_EACH_EDGE (e, ei, bb->preds) | |
4655 | { | |
4656 | tree val = e == e1 ? boolean_false_node : boolean_true_node; | |
4657 | add_phi_arg (phi, val, e, UNKNOWN_LOCATION); | |
4658 | } | |
4659 | m_gsi = gsi_for_stmt (final_stmt); | |
4660 | } | |
4661 | } | |
4662 | ||
4663 | finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt, MULT_EXPR); | |
4664 | } | |
4665 | ||
4666 | /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from | |
4667 | .{ADD,SUB,MUL}_OVERFLOW call. */ | |
4668 | ||
4669 | void | |
4670 | bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt) | |
4671 | { | |
4672 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
4673 | rhs1 = TREE_OPERAND (rhs1, 0); | |
4674 | if (obj == NULL_TREE) | |
4675 | { | |
4676 | int part = var_to_partition (m_map, gimple_assign_lhs (stmt)); | |
4677 | gcc_assert (m_vars[part] != NULL_TREE); | |
4678 | obj = m_vars[part]; | |
4679 | } | |
4680 | if (TREE_CODE (rhs1) == SSA_NAME | |
4681 | && (m_names == NULL | |
4682 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))) | |
4683 | { | |
4684 | lower_call (obj, SSA_NAME_DEF_STMT (rhs1)); | |
4685 | return; | |
4686 | } | |
4687 | int part = var_to_partition (m_map, rhs1); | |
4688 | gcc_assert (m_vars[part] != NULL_TREE); | |
4689 | tree var = m_vars[part]; | |
4690 | unsigned HOST_WIDE_INT nelts | |
4691 | = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec; | |
4692 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
4693 | if (!useless_type_conversion_p (atype, TREE_TYPE (obj))) | |
4694 | obj = build1 (VIEW_CONVERT_EXPR, atype, obj); | |
4695 | tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)), | |
4696 | gimple_assign_rhs_code (stmt) == REALPART_EXPR | |
4697 | ? 0 : nelts * m_limb_size); | |
4698 | tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off); | |
4699 | gimple *g = gimple_build_assign (obj, v2); | |
4700 | insert_before (g); | |
4701 | } | |
4702 | ||
4703 | /* Lower COMPLEX_EXPR stmt. */ | |
4704 | ||
4705 | void | |
4706 | bitint_large_huge::lower_complexexpr_stmt (gimple *stmt) | |
4707 | { | |
4708 | tree lhs = gimple_assign_lhs (stmt); | |
4709 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
4710 | tree rhs2 = gimple_assign_rhs2 (stmt); | |
4711 | int part = var_to_partition (m_map, lhs); | |
4712 | gcc_assert (m_vars[part] != NULL_TREE); | |
4713 | lhs = m_vars[part]; | |
4714 | unsigned HOST_WIDE_INT nelts | |
4715 | = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec; | |
4716 | tree atype = build_array_type_nelts (m_limb_type, nelts); | |
4717 | tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs))); | |
4718 | tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero); | |
4719 | tree v2; | |
4720 | if (TREE_CODE (rhs1) == SSA_NAME) | |
4721 | { | |
4722 | part = var_to_partition (m_map, rhs1); | |
4723 | gcc_assert (m_vars[part] != NULL_TREE); | |
4724 | v2 = m_vars[part]; | |
4725 | } | |
4726 | else if (integer_zerop (rhs1)) | |
4727 | v2 = build_zero_cst (atype); | |
4728 | else | |
4729 | v2 = tree_output_constant_def (rhs1); | |
4730 | if (!useless_type_conversion_p (atype, TREE_TYPE (v2))) | |
4731 | v2 = build1 (VIEW_CONVERT_EXPR, atype, v2); | |
4732 | gimple *g = gimple_build_assign (v1, v2); | |
4733 | insert_before (g); | |
4734 | tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)), | |
4735 | TYPE_SIZE_UNIT (atype)); | |
4736 | v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off); | |
4737 | if (TREE_CODE (rhs2) == SSA_NAME) | |
4738 | { | |
4739 | part = var_to_partition (m_map, rhs2); | |
4740 | gcc_assert (m_vars[part] != NULL_TREE); | |
4741 | v2 = m_vars[part]; | |
4742 | } | |
4743 | else if (integer_zerop (rhs2)) | |
4744 | v2 = build_zero_cst (atype); | |
4745 | else | |
4746 | v2 = tree_output_constant_def (rhs2); | |
4747 | if (!useless_type_conversion_p (atype, TREE_TYPE (v2))) | |
4748 | v2 = build1 (VIEW_CONVERT_EXPR, atype, v2); | |
4749 | g = gimple_build_assign (v1, v2); | |
4750 | insert_before (g); | |
4751 | } | |
4752 | ||
7383cb56 JJ |
4753 | /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt |
4754 | argument. */ | |
4755 | ||
4756 | void | |
4757 | bitint_large_huge::lower_bit_query (gimple *stmt) | |
4758 | { | |
4759 | tree arg0 = gimple_call_arg (stmt, 0); | |
4760 | tree arg1 = (gimple_call_num_args (stmt) == 2 | |
4761 | ? gimple_call_arg (stmt, 1) : NULL_TREE); | |
4762 | tree lhs = gimple_call_lhs (stmt); | |
4763 | gimple *g; | |
4764 | ||
4765 | if (!lhs) | |
4766 | { | |
4767 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
4768 | gsi_remove (&gsi, true); | |
4769 | return; | |
4770 | } | |
4771 | tree type = TREE_TYPE (arg0); | |
4772 | gcc_assert (TREE_CODE (type) == BITINT_TYPE); | |
4773 | bitint_prec_kind kind = bitint_precision_kind (type); | |
4774 | gcc_assert (kind >= bitint_prec_large); | |
4775 | enum internal_fn ifn = gimple_call_internal_fn (stmt); | |
4776 | enum built_in_function fcode = END_BUILTINS; | |
4777 | gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec | |
4778 | || TYPE_PRECISION (long_unsigned_type_node) == limb_prec | |
4779 | || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec); | |
4780 | switch (ifn) | |
4781 | { | |
4782 | case IFN_CLZ: | |
4783 | if (TYPE_PRECISION (unsigned_type_node) == limb_prec) | |
4784 | fcode = BUILT_IN_CLZ; | |
4785 | else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec) | |
4786 | fcode = BUILT_IN_CLZL; | |
4787 | else | |
4788 | fcode = BUILT_IN_CLZLL; | |
4789 | break; | |
4790 | case IFN_FFS: | |
4791 | /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood | |
4792 | we don't add the addend at the end. */ | |
4793 | arg1 = integer_zero_node; | |
4794 | /* FALLTHRU */ | |
4795 | case IFN_CTZ: | |
4796 | if (TYPE_PRECISION (unsigned_type_node) == limb_prec) | |
4797 | fcode = BUILT_IN_CTZ; | |
4798 | else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec) | |
4799 | fcode = BUILT_IN_CTZL; | |
4800 | else | |
4801 | fcode = BUILT_IN_CTZLL; | |
4802 | m_upwards = true; | |
4803 | break; | |
4804 | case IFN_CLRSB: | |
4805 | if (TYPE_PRECISION (unsigned_type_node) == limb_prec) | |
4806 | fcode = BUILT_IN_CLRSB; | |
4807 | else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec) | |
4808 | fcode = BUILT_IN_CLRSBL; | |
4809 | else | |
4810 | fcode = BUILT_IN_CLRSBLL; | |
4811 | break; | |
4812 | case IFN_PARITY: | |
4813 | if (TYPE_PRECISION (unsigned_type_node) == limb_prec) | |
4814 | fcode = BUILT_IN_PARITY; | |
4815 | else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec) | |
4816 | fcode = BUILT_IN_PARITYL; | |
4817 | else | |
4818 | fcode = BUILT_IN_PARITYLL; | |
4819 | m_upwards = true; | |
4820 | break; | |
4821 | case IFN_POPCOUNT: | |
4822 | if (TYPE_PRECISION (unsigned_type_node) == limb_prec) | |
4823 | fcode = BUILT_IN_POPCOUNT; | |
4824 | else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec) | |
4825 | fcode = BUILT_IN_POPCOUNTL; | |
4826 | else | |
4827 | fcode = BUILT_IN_POPCOUNTLL; | |
4828 | m_upwards = true; | |
4829 | break; | |
4830 | default: | |
4831 | gcc_unreachable (); | |
4832 | } | |
4833 | tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE; | |
4834 | unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type); | |
4835 | struct bq_details { edge e; tree val, addend; } *bqp = NULL; | |
4836 | basic_block edge_bb = NULL; | |
4837 | if (m_upwards) | |
4838 | { | |
4839 | tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE; | |
4840 | if (kind == bitint_prec_large) | |
4841 | cnt = CEIL (prec, limb_prec); | |
4842 | else | |
4843 | { | |
4844 | rem = (prec % (2 * limb_prec)); | |
4845 | end = (prec - rem) / limb_prec; | |
4846 | cnt = 2 + CEIL (rem, limb_prec); | |
4847 | idx = idx_first = create_loop (size_zero_node, &idx_next); | |
4848 | } | |
4849 | ||
4850 | if (ifn == IFN_CTZ || ifn == IFN_FFS) | |
4851 | { | |
4852 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
4853 | gsi_prev (&gsi); | |
4854 | edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
4855 | edge_bb = e->src; | |
4856 | if (kind == bitint_prec_large) | |
9a96a9e4 | 4857 | m_gsi = gsi_end_bb (edge_bb); |
7383cb56 JJ |
4858 | bqp = XALLOCAVEC (struct bq_details, cnt); |
4859 | } | |
4860 | else | |
4861 | m_after_stmt = stmt; | |
4862 | if (kind != bitint_prec_large) | |
4863 | m_upwards_2limb = end; | |
4864 | ||
4865 | for (unsigned i = 0; i < cnt; i++) | |
4866 | { | |
4867 | m_data_cnt = 0; | |
4868 | if (kind == bitint_prec_large) | |
4869 | idx = size_int (i); | |
4870 | else if (i >= 2) | |
4871 | idx = size_int (end + (i > 2)); | |
4872 | ||
4873 | tree rhs1 = handle_operand (arg0, idx); | |
4874 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1))) | |
4875 | { | |
4876 | if (!TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
4877 | rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1); | |
4878 | rhs1 = add_cast (m_limb_type, rhs1); | |
4879 | } | |
4880 | ||
4881 | tree in, out, tem; | |
4882 | if (ifn == IFN_PARITY) | |
4883 | in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out); | |
4884 | else if (ifn == IFN_FFS) | |
4885 | in = prepare_data_in_out (integer_one_node, idx, &out); | |
4886 | else | |
4887 | in = prepare_data_in_out (integer_zero_node, idx, &out); | |
4888 | ||
4889 | switch (ifn) | |
4890 | { | |
4891 | case IFN_CTZ: | |
4892 | case IFN_FFS: | |
4893 | g = gimple_build_cond (NE_EXPR, rhs1, | |
4894 | build_zero_cst (m_limb_type), | |
4895 | NULL_TREE, NULL_TREE); | |
4896 | insert_before (g); | |
4897 | edge e1, e2; | |
4898 | e1 = split_block (gsi_bb (m_gsi), g); | |
4899 | e1->flags = EDGE_FALSE_VALUE; | |
4900 | e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE); | |
4901 | e1->probability = profile_probability::unlikely (); | |
4902 | e2->probability = e1->probability.invert (); | |
4903 | if (i == 0) | |
4904 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
4905 | m_gsi = gsi_after_labels (e1->dest); | |
4906 | bqp[i].e = e2; | |
4907 | bqp[i].val = rhs1; | |
4908 | if (tree_fits_uhwi_p (idx)) | |
4909 | bqp[i].addend | |
4910 | = build_int_cst (integer_type_node, | |
4911 | tree_to_uhwi (idx) * limb_prec | |
4912 | + (ifn == IFN_FFS)); | |
4913 | else | |
4914 | { | |
4915 | bqp[i].addend = in; | |
4916 | if (i == 1) | |
4917 | res = out; | |
4918 | else | |
4919 | res = make_ssa_name (integer_type_node); | |
4920 | g = gimple_build_assign (res, PLUS_EXPR, in, | |
4921 | build_int_cst (integer_type_node, | |
4922 | limb_prec)); | |
4923 | insert_before (g); | |
4924 | m_data[m_data_cnt] = res; | |
4925 | } | |
4926 | break; | |
4927 | case IFN_PARITY: | |
4928 | if (!integer_zerop (in)) | |
4929 | { | |
4930 | if (kind == bitint_prec_huge && i == 1) | |
4931 | res = out; | |
4932 | else | |
4933 | res = make_ssa_name (m_limb_type); | |
4934 | g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1); | |
4935 | insert_before (g); | |
4936 | } | |
4937 | else | |
4938 | res = rhs1; | |
4939 | m_data[m_data_cnt] = res; | |
4940 | break; | |
4941 | case IFN_POPCOUNT: | |
4942 | g = gimple_build_call (fndecl, 1, rhs1); | |
4943 | tem = make_ssa_name (integer_type_node); | |
4944 | gimple_call_set_lhs (g, tem); | |
4945 | insert_before (g); | |
4946 | if (!integer_zerop (in)) | |
4947 | { | |
4948 | if (kind == bitint_prec_huge && i == 1) | |
4949 | res = out; | |
4950 | else | |
4951 | res = make_ssa_name (integer_type_node); | |
4952 | g = gimple_build_assign (res, PLUS_EXPR, in, tem); | |
4953 | insert_before (g); | |
4954 | } | |
4955 | else | |
4956 | res = tem; | |
4957 | m_data[m_data_cnt] = res; | |
4958 | break; | |
4959 | default: | |
4960 | gcc_unreachable (); | |
4961 | } | |
4962 | ||
4963 | m_first = false; | |
4964 | if (kind == bitint_prec_huge && i <= 1) | |
4965 | { | |
4966 | if (i == 0) | |
4967 | { | |
4968 | idx = make_ssa_name (sizetype); | |
4969 | g = gimple_build_assign (idx, PLUS_EXPR, idx_first, | |
4970 | size_one_node); | |
4971 | insert_before (g); | |
4972 | } | |
4973 | else | |
4974 | { | |
4975 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first, | |
4976 | size_int (2)); | |
4977 | insert_before (g); | |
4978 | g = gimple_build_cond (NE_EXPR, idx_next, size_int (end), | |
4979 | NULL_TREE, NULL_TREE); | |
4980 | insert_before (g); | |
4981 | if (ifn == IFN_CTZ || ifn == IFN_FFS) | |
4982 | m_gsi = gsi_after_labels (edge_bb); | |
4983 | else | |
4984 | m_gsi = gsi_for_stmt (stmt); | |
be9e8de6 | 4985 | m_bb = NULL; |
7383cb56 JJ |
4986 | } |
4987 | } | |
4988 | } | |
4989 | } | |
4990 | else | |
4991 | { | |
4992 | tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE; | |
4993 | int sub_one = 0; | |
4994 | if (kind == bitint_prec_large) | |
4995 | cnt = CEIL (prec, limb_prec); | |
4996 | else | |
4997 | { | |
4998 | rem = prec % limb_prec; | |
4999 | if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB)) | |
5000 | rem = limb_prec; | |
5001 | end = (prec - rem) / limb_prec; | |
5002 | cnt = 1 + (rem != 0); | |
5003 | if (ifn == IFN_CLRSB) | |
5004 | sub_one = 1; | |
5005 | } | |
5006 | ||
5007 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
5008 | gsi_prev (&gsi); | |
5009 | edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
5010 | edge_bb = e->src; | |
9a96a9e4 | 5011 | m_gsi = gsi_end_bb (edge_bb); |
7383cb56 JJ |
5012 | |
5013 | if (ifn == IFN_CLZ) | |
5014 | bqp = XALLOCAVEC (struct bq_details, cnt); | |
5015 | else | |
5016 | { | |
5017 | gsi = gsi_for_stmt (stmt); | |
5018 | gsi_prev (&gsi); | |
5019 | e = split_block (gsi_bb (gsi), gsi_stmt (gsi)); | |
5020 | edge_bb = e->src; | |
5021 | bqp = XALLOCAVEC (struct bq_details, 2 * cnt); | |
5022 | } | |
5023 | ||
5024 | for (unsigned i = 0; i < cnt; i++) | |
5025 | { | |
5026 | m_data_cnt = 0; | |
5027 | if (kind == bitint_prec_large) | |
5028 | idx = size_int (cnt - i - 1); | |
5029 | else if (i == cnt - 1) | |
5030 | idx = create_loop (size_int (end - 1), &idx_next); | |
5031 | else | |
5032 | idx = size_int (end); | |
5033 | ||
5034 | tree rhs1 = handle_operand (arg0, idx); | |
5035 | if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1))) | |
5036 | { | |
5037 | if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
5038 | rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1); | |
5039 | else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1))) | |
5040 | rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1); | |
5041 | rhs1 = add_cast (m_limb_type, rhs1); | |
5042 | } | |
5043 | ||
5044 | if (ifn == IFN_CLZ) | |
5045 | { | |
5046 | g = gimple_build_cond (NE_EXPR, rhs1, | |
5047 | build_zero_cst (m_limb_type), | |
5048 | NULL_TREE, NULL_TREE); | |
5049 | insert_before (g); | |
5050 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
5051 | e1->flags = EDGE_FALSE_VALUE; | |
5052 | edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE); | |
5053 | e1->probability = profile_probability::unlikely (); | |
5054 | e2->probability = e1->probability.invert (); | |
5055 | if (i == 0) | |
5056 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
5057 | m_gsi = gsi_after_labels (e1->dest); | |
5058 | bqp[i].e = e2; | |
5059 | bqp[i].val = rhs1; | |
5060 | } | |
5061 | else | |
5062 | { | |
5063 | if (i == 0) | |
5064 | { | |
5065 | first = rhs1; | |
5066 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
5067 | PLUS_EXPR, rhs1, | |
5068 | build_int_cst (m_limb_type, 1)); | |
5069 | insert_before (g); | |
5070 | g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g), | |
5071 | build_int_cst (m_limb_type, 1), | |
5072 | NULL_TREE, NULL_TREE); | |
5073 | insert_before (g); | |
5074 | } | |
5075 | else | |
5076 | { | |
5077 | g = gimple_build_assign (make_ssa_name (m_limb_type), | |
5078 | BIT_XOR_EXPR, rhs1, first); | |
5079 | insert_before (g); | |
5080 | tree stype = signed_type_for (m_limb_type); | |
5081 | g = gimple_build_cond (LT_EXPR, | |
5082 | add_cast (stype, | |
5083 | gimple_assign_lhs (g)), | |
5084 | build_zero_cst (stype), | |
5085 | NULL_TREE, NULL_TREE); | |
5086 | insert_before (g); | |
5087 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
5088 | e1->flags = EDGE_FALSE_VALUE; | |
5089 | edge e2 = make_edge (e1->src, gimple_bb (stmt), | |
5090 | EDGE_TRUE_VALUE); | |
5091 | e1->probability = profile_probability::unlikely (); | |
5092 | e2->probability = e1->probability.invert (); | |
5093 | if (i == 1) | |
5094 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, | |
5095 | e2->src); | |
5096 | m_gsi = gsi_after_labels (e1->dest); | |
5097 | bqp[2 * i].e = e2; | |
5098 | g = gimple_build_cond (NE_EXPR, rhs1, first, | |
5099 | NULL_TREE, NULL_TREE); | |
5100 | insert_before (g); | |
5101 | } | |
5102 | edge e1 = split_block (gsi_bb (m_gsi), g); | |
5103 | e1->flags = EDGE_FALSE_VALUE; | |
5104 | edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE); | |
5105 | e1->probability = profile_probability::unlikely (); | |
5106 | e2->probability = e1->probability.invert (); | |
5107 | if (i == 0) | |
5108 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src); | |
5109 | m_gsi = gsi_after_labels (e1->dest); | |
5110 | bqp[2 * i + 1].e = e2; | |
5111 | bqp[i].val = rhs1; | |
5112 | } | |
5113 | if (tree_fits_uhwi_p (idx)) | |
5114 | bqp[i].addend | |
5115 | = build_int_cst (integer_type_node, | |
5116 | (int) prec | |
5117 | - (((int) tree_to_uhwi (idx) + 1) | |
5118 | * limb_prec) - sub_one); | |
5119 | else | |
5120 | { | |
5121 | tree in, out; | |
5122 | in = build_int_cst (integer_type_node, rem - sub_one); | |
5123 | m_first = true; | |
5124 | in = prepare_data_in_out (in, idx, &out); | |
5125 | out = m_data[m_data_cnt + 1]; | |
5126 | bqp[i].addend = in; | |
5127 | g = gimple_build_assign (out, PLUS_EXPR, in, | |
5128 | build_int_cst (integer_type_node, | |
5129 | limb_prec)); | |
5130 | insert_before (g); | |
5131 | m_data[m_data_cnt] = out; | |
5132 | } | |
5133 | ||
5134 | m_first = false; | |
5135 | if (kind == bitint_prec_huge && i == cnt - 1) | |
5136 | { | |
5137 | g = gimple_build_assign (idx_next, PLUS_EXPR, idx, | |
5138 | size_int (-1)); | |
5139 | insert_before (g); | |
5140 | g = gimple_build_cond (NE_EXPR, idx, size_zero_node, | |
5141 | NULL_TREE, NULL_TREE); | |
5142 | insert_before (g); | |
5143 | edge true_edge, false_edge; | |
5144 | extract_true_false_edges_from_block (gsi_bb (m_gsi), | |
5145 | &true_edge, &false_edge); | |
5146 | m_gsi = gsi_after_labels (false_edge->dest); | |
be9e8de6 | 5147 | m_bb = NULL; |
7383cb56 JJ |
5148 | } |
5149 | } | |
5150 | } | |
5151 | switch (ifn) | |
5152 | { | |
5153 | case IFN_CLZ: | |
5154 | case IFN_CTZ: | |
5155 | case IFN_FFS: | |
5156 | gphi *phi1, *phi2, *phi3; | |
5157 | basic_block bb; | |
5158 | bb = gsi_bb (m_gsi); | |
5159 | remove_edge (find_edge (bb, gimple_bb (stmt))); | |
5160 | phi1 = create_phi_node (make_ssa_name (m_limb_type), | |
5161 | gimple_bb (stmt)); | |
5162 | phi2 = create_phi_node (make_ssa_name (integer_type_node), | |
5163 | gimple_bb (stmt)); | |
5164 | for (unsigned i = 0; i < cnt; i++) | |
5165 | { | |
5166 | add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION); | |
5167 | add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION); | |
5168 | } | |
5169 | if (arg1 == NULL_TREE) | |
5170 | { | |
5171 | g = gimple_build_builtin_unreachable (m_loc); | |
5172 | insert_before (g); | |
5173 | } | |
5174 | m_gsi = gsi_for_stmt (stmt); | |
5175 | g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1)); | |
5176 | gimple_call_set_lhs (g, make_ssa_name (integer_type_node)); | |
5177 | insert_before (g); | |
5178 | if (arg1 == NULL_TREE) | |
5179 | g = gimple_build_assign (lhs, PLUS_EXPR, | |
5180 | gimple_phi_result (phi2), | |
5181 | gimple_call_lhs (g)); | |
5182 | else | |
5183 | { | |
5184 | g = gimple_build_assign (make_ssa_name (integer_type_node), | |
5185 | PLUS_EXPR, gimple_phi_result (phi2), | |
5186 | gimple_call_lhs (g)); | |
5187 | insert_before (g); | |
5188 | edge e1 = split_block (gimple_bb (stmt), g); | |
5189 | edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU); | |
5190 | e2->probability = profile_probability::always (); | |
5191 | set_immediate_dominator (CDI_DOMINATORS, e1->dest, | |
5192 | get_immediate_dominator (CDI_DOMINATORS, | |
5193 | e1->src)); | |
5194 | phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest); | |
5195 | add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION); | |
5196 | add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION); | |
5197 | m_gsi = gsi_for_stmt (stmt); | |
5198 | g = gimple_build_assign (lhs, gimple_phi_result (phi3)); | |
5199 | } | |
5200 | gsi_replace (&m_gsi, g, true); | |
5201 | break; | |
5202 | case IFN_CLRSB: | |
5203 | bb = gsi_bb (m_gsi); | |
5204 | remove_edge (find_edge (bb, edge_bb)); | |
5205 | edge e; | |
5206 | e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU); | |
5207 | e->probability = profile_probability::always (); | |
5208 | set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt), | |
5209 | get_immediate_dominator (CDI_DOMINATORS, | |
5210 | edge_bb)); | |
5211 | phi1 = create_phi_node (make_ssa_name (m_limb_type), | |
5212 | edge_bb); | |
5213 | phi2 = create_phi_node (make_ssa_name (integer_type_node), | |
5214 | edge_bb); | |
5215 | phi3 = create_phi_node (make_ssa_name (integer_type_node), | |
5216 | gimple_bb (stmt)); | |
5217 | for (unsigned i = 0; i < cnt; i++) | |
5218 | { | |
5219 | add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION); | |
5220 | add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e, | |
5221 | UNKNOWN_LOCATION); | |
5222 | tree a = bqp[i].addend; | |
5223 | if (i && kind == bitint_prec_large) | |
5224 | a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node); | |
5225 | if (i) | |
5226 | add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION); | |
5227 | } | |
5228 | add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e, | |
5229 | UNKNOWN_LOCATION); | |
5230 | m_gsi = gsi_after_labels (edge_bb); | |
5231 | g = gimple_build_call (fndecl, 1, | |
5232 | add_cast (signed_type_for (m_limb_type), | |
5233 | gimple_phi_result (phi1))); | |
5234 | gimple_call_set_lhs (g, make_ssa_name (integer_type_node)); | |
5235 | insert_before (g); | |
5236 | g = gimple_build_assign (make_ssa_name (integer_type_node), | |
5237 | PLUS_EXPR, gimple_call_lhs (g), | |
5238 | gimple_phi_result (phi2)); | |
5239 | insert_before (g); | |
5240 | if (kind != bitint_prec_large) | |
5241 | { | |
5242 | g = gimple_build_assign (make_ssa_name (integer_type_node), | |
5243 | PLUS_EXPR, gimple_assign_lhs (g), | |
5244 | integer_one_node); | |
5245 | insert_before (g); | |
5246 | } | |
5247 | add_phi_arg (phi3, gimple_assign_lhs (g), | |
5248 | find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION); | |
5249 | m_gsi = gsi_for_stmt (stmt); | |
5250 | g = gimple_build_assign (lhs, gimple_phi_result (phi3)); | |
5251 | gsi_replace (&m_gsi, g, true); | |
5252 | break; | |
5253 | case IFN_PARITY: | |
5254 | g = gimple_build_call (fndecl, 1, res); | |
5255 | gimple_call_set_lhs (g, lhs); | |
5256 | gsi_replace (&m_gsi, g, true); | |
5257 | break; | |
5258 | case IFN_POPCOUNT: | |
5259 | g = gimple_build_assign (lhs, res); | |
5260 | gsi_replace (&m_gsi, g, true); | |
5261 | break; | |
5262 | default: | |
5263 | gcc_unreachable (); | |
5264 | } | |
5265 | } | |
5266 | ||
a9d6c7fb JJ |
5267 | /* Lower a call statement with one or more large/huge _BitInt |
5268 | arguments or large/huge _BitInt return value. */ | |
5269 | ||
5270 | void | |
5271 | bitint_large_huge::lower_call (tree obj, gimple *stmt) | |
5272 | { | |
5273 | gimple_stmt_iterator gsi = gsi_for_stmt (stmt); | |
5274 | unsigned int nargs = gimple_call_num_args (stmt); | |
5275 | if (gimple_call_internal_p (stmt)) | |
5276 | switch (gimple_call_internal_fn (stmt)) | |
5277 | { | |
5278 | case IFN_ADD_OVERFLOW: | |
5279 | case IFN_SUB_OVERFLOW: | |
5280 | case IFN_UBSAN_CHECK_ADD: | |
5281 | case IFN_UBSAN_CHECK_SUB: | |
5282 | lower_addsub_overflow (obj, stmt); | |
5283 | return; | |
5284 | case IFN_MUL_OVERFLOW: | |
5285 | case IFN_UBSAN_CHECK_MUL: | |
5286 | lower_mul_overflow (obj, stmt); | |
5287 | return; | |
7383cb56 JJ |
5288 | case IFN_CLZ: |
5289 | case IFN_CTZ: | |
5290 | case IFN_CLRSB: | |
5291 | case IFN_FFS: | |
5292 | case IFN_PARITY: | |
5293 | case IFN_POPCOUNT: | |
5294 | lower_bit_query (stmt); | |
5295 | return; | |
a9d6c7fb JJ |
5296 | default: |
5297 | break; | |
5298 | } | |
90b98723 | 5299 | bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0; |
a9d6c7fb JJ |
5300 | for (unsigned int i = 0; i < nargs; ++i) |
5301 | { | |
5302 | tree arg = gimple_call_arg (stmt, i); | |
5303 | if (TREE_CODE (arg) != SSA_NAME | |
5304 | || TREE_CODE (TREE_TYPE (arg)) != BITINT_TYPE | |
5305 | || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle) | |
5306 | continue; | |
c1680bd2 JJ |
5307 | if (SSA_NAME_IS_DEFAULT_DEF (arg) |
5308 | && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg)))) | |
5309 | { | |
5310 | tree var = create_tmp_reg (TREE_TYPE (arg)); | |
5311 | arg = get_or_create_ssa_default_def (cfun, var); | |
5312 | } | |
5313 | else | |
5314 | { | |
5315 | int p = var_to_partition (m_map, arg); | |
5316 | tree v = m_vars[p]; | |
5317 | gcc_assert (v != NULL_TREE); | |
5318 | if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v))) | |
5319 | v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v); | |
5320 | arg = make_ssa_name (TREE_TYPE (arg)); | |
5321 | gimple *g = gimple_build_assign (arg, v); | |
5322 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
c9e94ae4 | 5323 | if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt))) |
90b98723 JJ |
5324 | { |
5325 | m_returns_twice_calls.safe_push (stmt); | |
5326 | returns_twice = false; | |
5327 | } | |
c1680bd2 | 5328 | } |
a9d6c7fb JJ |
5329 | gimple_call_set_arg (stmt, i, arg); |
5330 | if (m_preserved == NULL) | |
5331 | m_preserved = BITMAP_ALLOC (NULL); | |
5332 | bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg)); | |
5333 | } | |
5334 | tree lhs = gimple_call_lhs (stmt); | |
5335 | if (lhs | |
5336 | && TREE_CODE (lhs) == SSA_NAME | |
5337 | && TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE | |
5338 | && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large) | |
5339 | { | |
5340 | int p = var_to_partition (m_map, lhs); | |
5341 | tree v = m_vars[p]; | |
5342 | gcc_assert (v != NULL_TREE); | |
5343 | if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v))) | |
5344 | v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v); | |
5345 | gimple_call_set_lhs (stmt, v); | |
5346 | SSA_NAME_DEF_STMT (lhs) = gimple_build_nop (); | |
5347 | } | |
5348 | update_stmt (stmt); | |
5349 | } | |
5350 | ||
5351 | /* Lower __asm STMT which involves large/huge _BitInt values. */ | |
5352 | ||
5353 | void | |
5354 | bitint_large_huge::lower_asm (gimple *stmt) | |
5355 | { | |
5356 | gasm *g = as_a <gasm *> (stmt); | |
5357 | unsigned noutputs = gimple_asm_noutputs (g); | |
5358 | unsigned ninputs = gimple_asm_ninputs (g); | |
5359 | ||
5360 | for (unsigned i = 0; i < noutputs; ++i) | |
5361 | { | |
5362 | tree t = gimple_asm_output_op (g, i); | |
5363 | tree s = TREE_VALUE (t); | |
5364 | if (TREE_CODE (s) == SSA_NAME | |
5365 | && TREE_CODE (TREE_TYPE (s)) == BITINT_TYPE | |
5366 | && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large) | |
5367 | { | |
5368 | int part = var_to_partition (m_map, s); | |
5369 | gcc_assert (m_vars[part] != NULL_TREE); | |
5370 | TREE_VALUE (t) = m_vars[part]; | |
5371 | } | |
5372 | } | |
5373 | for (unsigned i = 0; i < ninputs; ++i) | |
5374 | { | |
5375 | tree t = gimple_asm_input_op (g, i); | |
5376 | tree s = TREE_VALUE (t); | |
5377 | if (TREE_CODE (s) == SSA_NAME | |
5378 | && TREE_CODE (TREE_TYPE (s)) == BITINT_TYPE | |
5379 | && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large) | |
5380 | { | |
49e75666 JJ |
5381 | if (SSA_NAME_IS_DEFAULT_DEF (s) |
5382 | && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s)))) | |
5383 | { | |
5384 | TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint"); | |
5385 | mark_addressable (TREE_VALUE (t)); | |
5386 | } | |
5387 | else | |
5388 | { | |
5389 | int part = var_to_partition (m_map, s); | |
5390 | gcc_assert (m_vars[part] != NULL_TREE); | |
5391 | TREE_VALUE (t) = m_vars[part]; | |
5392 | } | |
a9d6c7fb JJ |
5393 | } |
5394 | } | |
5395 | update_stmt (stmt); | |
5396 | } | |
5397 | ||
5398 | /* Lower statement STMT which involves large/huge _BitInt values | |
5399 | into code accessing individual limbs. */ | |
5400 | ||
5401 | void | |
5402 | bitint_large_huge::lower_stmt (gimple *stmt) | |
5403 | { | |
5404 | m_first = true; | |
5405 | m_lhs = NULL_TREE; | |
5406 | m_data.truncate (0); | |
5407 | m_data_cnt = 0; | |
5408 | m_gsi = gsi_for_stmt (stmt); | |
5409 | m_after_stmt = NULL; | |
5410 | m_bb = NULL; | |
5411 | m_init_gsi = m_gsi; | |
5412 | gsi_prev (&m_init_gsi); | |
5413 | m_preheader_bb = NULL; | |
5414 | m_upwards_2limb = 0; | |
5415 | m_upwards = false; | |
5416 | m_var_msb = false; | |
5417 | m_cast_conditional = false; | |
5418 | m_bitfld_load = 0; | |
5419 | m_loc = gimple_location (stmt); | |
5420 | if (is_gimple_call (stmt)) | |
5421 | { | |
5422 | lower_call (NULL_TREE, stmt); | |
5423 | return; | |
5424 | } | |
5425 | if (gimple_code (stmt) == GIMPLE_ASM) | |
5426 | { | |
5427 | lower_asm (stmt); | |
5428 | return; | |
5429 | } | |
5430 | tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE; | |
5431 | tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2); | |
5432 | bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR); | |
5433 | bool mergeable_cast_p = false; | |
5434 | bool final_cast_p = false; | |
5435 | if (gimple_assign_cast_p (stmt)) | |
5436 | { | |
5437 | lhs = gimple_assign_lhs (stmt); | |
5438 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
c9bdcb0c JJ |
5439 | if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR) |
5440 | rhs1 = TREE_OPERAND (rhs1, 0); | |
a9d6c7fb JJ |
5441 | if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE |
5442 | && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large | |
5443 | && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))) | |
5444 | mergeable_cast_p = true; | |
5445 | else if (TREE_CODE (TREE_TYPE (rhs1)) == BITINT_TYPE | |
5446 | && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large | |
fb28d5cd | 5447 | && (INTEGRAL_TYPE_P (TREE_TYPE (lhs)) |
5e7a176e JJ |
5448 | || POINTER_TYPE_P (TREE_TYPE (lhs)) |
5449 | || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)) | |
a9d6c7fb JJ |
5450 | { |
5451 | final_cast_p = true; | |
5e7a176e JJ |
5452 | if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE |
5453 | && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE) | |
5454 | || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)) | |
5455 | && !POINTER_TYPE_P (TREE_TYPE (lhs)))) | |
c9bdcb0c JJ |
5456 | && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR) |
5457 | { | |
5458 | /* Handle VIEW_CONVERT_EXPRs to not generally supported | |
5459 | huge INTEGER_TYPEs like uint256_t or uint512_t. These | |
5460 | are usually emitted from memcpy folding and backends | |
5e7a176e JJ |
5461 | support moves with them but that is usually it. |
5462 | Similarly handle VCEs to vector/complex types etc. */ | |
c9bdcb0c JJ |
5463 | gcc_assert (TREE_CODE (rhs1) == SSA_NAME); |
5464 | if (SSA_NAME_IS_DEFAULT_DEF (rhs1) | |
5465 | && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1)))) | |
5466 | { | |
5467 | tree var = create_tmp_reg (TREE_TYPE (lhs)); | |
5468 | rhs1 = get_or_create_ssa_default_def (cfun, var); | |
5469 | gimple_assign_set_rhs1 (stmt, rhs1); | |
5470 | gimple_assign_set_rhs_code (stmt, SSA_NAME); | |
5471 | } | |
d3d0fcb6 JJ |
5472 | else if (m_names == NULL |
5473 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))) | |
5474 | { | |
5475 | gimple *g = SSA_NAME_DEF_STMT (rhs1); | |
5476 | gcc_assert (gimple_assign_load_p (g)); | |
5477 | tree mem = gimple_assign_rhs1 (g); | |
5478 | tree ltype = TREE_TYPE (lhs); | |
5479 | addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem)); | |
5480 | if (as != TYPE_ADDR_SPACE (ltype)) | |
5481 | ltype | |
5482 | = build_qualified_type (ltype, | |
5483 | TYPE_QUALS (ltype) | |
5484 | | ENCODE_QUAL_ADDR_SPACE (as)); | |
c27148f2 | 5485 | rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem)); |
d3d0fcb6 JJ |
5486 | gimple_assign_set_rhs1 (stmt, rhs1); |
5487 | } | |
c9bdcb0c JJ |
5488 | else |
5489 | { | |
5490 | int part = var_to_partition (m_map, rhs1); | |
5491 | gcc_assert (m_vars[part] != NULL_TREE); | |
5492 | rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), | |
5493 | m_vars[part]); | |
5494 | gimple_assign_set_rhs1 (stmt, rhs1); | |
5495 | } | |
5496 | update_stmt (stmt); | |
5497 | return; | |
5498 | } | |
a9d6c7fb JJ |
5499 | if (TREE_CODE (rhs1) == SSA_NAME |
5500 | && (m_names == NULL | |
5501 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))) | |
5502 | { | |
5503 | gimple *g = SSA_NAME_DEF_STMT (rhs1); | |
5504 | if (is_gimple_assign (g) | |
5505 | && gimple_assign_rhs_code (g) == IMAGPART_EXPR) | |
5506 | { | |
5507 | tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0); | |
5508 | if (TREE_CODE (rhs2) == SSA_NAME | |
5509 | && (m_names == NULL | |
5510 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2)))) | |
5511 | { | |
5512 | g = SSA_NAME_DEF_STMT (rhs2); | |
5513 | int ovf = optimizable_arith_overflow (g); | |
5514 | if (ovf == 2) | |
5515 | /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR | |
5516 | and IMAGPART_EXPR uses, where the latter is cast to | |
5517 | non-_BitInt, it will be optimized when handling | |
5518 | the REALPART_EXPR. */ | |
5519 | return; | |
5520 | if (ovf == 1) | |
5521 | { | |
5522 | lower_call (NULL_TREE, g); | |
5523 | return; | |
5524 | } | |
5525 | } | |
5526 | } | |
5527 | } | |
5528 | } | |
5e7a176e JJ |
5529 | else if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE |
5530 | && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large | |
5531 | && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1)) | |
5532 | && !POINTER_TYPE_P (TREE_TYPE (rhs1)) | |
5533 | && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR) | |
5534 | { | |
5535 | int part = var_to_partition (m_map, lhs); | |
5536 | gcc_assert (m_vars[part] != NULL_TREE); | |
5537 | lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]); | |
5538 | insert_before (gimple_build_assign (lhs, rhs1)); | |
5539 | return; | |
5540 | } | |
a9d6c7fb JJ |
5541 | } |
5542 | if (gimple_store_p (stmt)) | |
5543 | { | |
5544 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
5545 | if (TREE_CODE (rhs1) == SSA_NAME | |
5546 | && (m_names == NULL | |
5547 | || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))) | |
5548 | { | |
5549 | gimple *g = SSA_NAME_DEF_STMT (rhs1); | |
5550 | m_loc = gimple_location (g); | |
5551 | lhs = gimple_assign_lhs (stmt); | |
5552 | if (is_gimple_assign (g) && !mergeable_op (g)) | |
5553 | switch (gimple_assign_rhs_code (g)) | |
5554 | { | |
5555 | case LSHIFT_EXPR: | |
5556 | case RSHIFT_EXPR: | |
5557 | lower_shift_stmt (lhs, g); | |
5558 | handled: | |
5559 | m_gsi = gsi_for_stmt (stmt); | |
5560 | unlink_stmt_vdef (stmt); | |
5561 | release_ssa_name (gimple_vdef (stmt)); | |
5562 | gsi_remove (&m_gsi, true); | |
5563 | return; | |
5564 | case MULT_EXPR: | |
5565 | case TRUNC_DIV_EXPR: | |
5566 | case TRUNC_MOD_EXPR: | |
5567 | lower_muldiv_stmt (lhs, g); | |
5568 | goto handled; | |
5569 | case FIX_TRUNC_EXPR: | |
5570 | lower_float_conv_stmt (lhs, g); | |
5571 | goto handled; | |
5572 | case REALPART_EXPR: | |
5573 | case IMAGPART_EXPR: | |
5574 | lower_cplxpart_stmt (lhs, g); | |
5575 | goto handled; | |
5e7a176e JJ |
5576 | case VIEW_CONVERT_EXPR: |
5577 | { | |
5578 | tree rhs1 = gimple_assign_rhs1 (g); | |
5579 | rhs1 = TREE_OPERAND (rhs1, 0); | |
5580 | if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1)) | |
5581 | && !POINTER_TYPE_P (TREE_TYPE (rhs1))) | |
5582 | { | |
5583 | tree ltype = TREE_TYPE (rhs1); | |
5584 | addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs)); | |
5585 | ltype | |
5586 | = build_qualified_type (ltype, | |
5587 | TYPE_QUALS (TREE_TYPE (lhs)) | |
5588 | | ENCODE_QUAL_ADDR_SPACE (as)); | |
5589 | lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs); | |
5590 | gimple_assign_set_lhs (stmt, lhs); | |
5591 | gimple_assign_set_rhs1 (stmt, rhs1); | |
5592 | gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1)); | |
5593 | update_stmt (stmt); | |
5594 | return; | |
5595 | } | |
5596 | } | |
5597 | break; | |
a9d6c7fb JJ |
5598 | default: |
5599 | break; | |
5600 | } | |
5601 | else if (optimizable_arith_overflow (g) == 3) | |
5602 | { | |
5603 | lower_call (lhs, g); | |
5604 | goto handled; | |
5605 | } | |
5606 | m_loc = gimple_location (stmt); | |
5607 | } | |
5608 | } | |
5609 | if (mergeable_op (stmt) | |
5610 | || gimple_store_p (stmt) | |
5611 | || gimple_assign_load_p (stmt) | |
5612 | || eq_p | |
5613 | || mergeable_cast_p) | |
5614 | { | |
5615 | lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2); | |
5616 | if (!eq_p) | |
5617 | return; | |
5618 | } | |
5619 | else if (cmp_code != ERROR_MARK) | |
5620 | lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2); | |
5621 | if (cmp_code != ERROR_MARK) | |
5622 | { | |
5623 | if (gimple_code (stmt) == GIMPLE_COND) | |
5624 | { | |
5625 | gcond *cstmt = as_a <gcond *> (stmt); | |
5626 | gimple_cond_set_lhs (cstmt, lhs); | |
5627 | gimple_cond_set_rhs (cstmt, boolean_false_node); | |
5628 | gimple_cond_set_code (cstmt, cmp_code); | |
5629 | update_stmt (stmt); | |
5630 | return; | |
5631 | } | |
5632 | if (gimple_assign_rhs_code (stmt) == COND_EXPR) | |
5633 | { | |
5634 | tree cond = build2 (cmp_code, boolean_type_node, lhs, | |
5635 | boolean_false_node); | |
5636 | gimple_assign_set_rhs1 (stmt, cond); | |
5637 | lhs = gimple_assign_lhs (stmt); | |
5638 | gcc_assert (TREE_CODE (TREE_TYPE (lhs)) != BITINT_TYPE | |
5639 | || (bitint_precision_kind (TREE_TYPE (lhs)) | |
5640 | <= bitint_prec_middle)); | |
5641 | update_stmt (stmt); | |
5642 | return; | |
5643 | } | |
5644 | gimple_assign_set_rhs1 (stmt, lhs); | |
5645 | gimple_assign_set_rhs2 (stmt, boolean_false_node); | |
5646 | gimple_assign_set_rhs_code (stmt, cmp_code); | |
5647 | update_stmt (stmt); | |
5648 | return; | |
5649 | } | |
5650 | if (final_cast_p) | |
5651 | { | |
5652 | tree lhs_type = TREE_TYPE (lhs); | |
5653 | /* Add support for 3 or more limbs filled in from normal integral | |
5654 | type if this assert fails. If no target chooses limb mode smaller | |
5655 | than half of largest supported normal integral type, this will not | |
5656 | be needed. */ | |
5657 | gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec); | |
5658 | gimple *g; | |
fb28d5cd JJ |
5659 | if ((TREE_CODE (lhs_type) == BITINT_TYPE |
5660 | && bitint_precision_kind (lhs_type) == bitint_prec_middle) | |
5661 | || POINTER_TYPE_P (lhs_type)) | |
a9d6c7fb JJ |
5662 | lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type), |
5663 | TYPE_UNSIGNED (lhs_type)); | |
5664 | m_data_cnt = 0; | |
5665 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
5666 | tree r1 = handle_operand (rhs1, size_int (0)); | |
5667 | if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1))) | |
5668 | r1 = add_cast (lhs_type, r1); | |
5669 | if (TYPE_PRECISION (lhs_type) > limb_prec) | |
5670 | { | |
5671 | m_data_cnt = 0; | |
5672 | m_first = false; | |
5673 | tree r2 = handle_operand (rhs1, size_int (1)); | |
5674 | r2 = add_cast (lhs_type, r2); | |
5675 | g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2, | |
5676 | build_int_cst (unsigned_type_node, | |
5677 | limb_prec)); | |
5678 | insert_before (g); | |
5679 | g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1, | |
5680 | gimple_assign_lhs (g)); | |
5681 | insert_before (g); | |
5682 | r1 = gimple_assign_lhs (g); | |
5683 | } | |
5684 | if (lhs_type != TREE_TYPE (lhs)) | |
5685 | g = gimple_build_assign (lhs, NOP_EXPR, r1); | |
5686 | else | |
5687 | g = gimple_build_assign (lhs, r1); | |
5688 | gsi_replace (&m_gsi, g, true); | |
5689 | return; | |
5690 | } | |
5691 | if (is_gimple_assign (stmt)) | |
5692 | switch (gimple_assign_rhs_code (stmt)) | |
5693 | { | |
5694 | case LSHIFT_EXPR: | |
5695 | case RSHIFT_EXPR: | |
5696 | lower_shift_stmt (NULL_TREE, stmt); | |
5697 | return; | |
5698 | case MULT_EXPR: | |
5699 | case TRUNC_DIV_EXPR: | |
5700 | case TRUNC_MOD_EXPR: | |
5701 | lower_muldiv_stmt (NULL_TREE, stmt); | |
5702 | return; | |
5703 | case FIX_TRUNC_EXPR: | |
5704 | case FLOAT_EXPR: | |
5705 | lower_float_conv_stmt (NULL_TREE, stmt); | |
5706 | return; | |
5707 | case REALPART_EXPR: | |
5708 | case IMAGPART_EXPR: | |
5709 | lower_cplxpart_stmt (NULL_TREE, stmt); | |
5710 | return; | |
5711 | case COMPLEX_EXPR: | |
5712 | lower_complexexpr_stmt (stmt); | |
5713 | return; | |
5714 | default: | |
5715 | break; | |
5716 | } | |
5717 | gcc_unreachable (); | |
5718 | } | |
5719 | ||
5720 | /* Helper for walk_non_aliased_vuses. Determine if we arrived at | |
5721 | the desired memory state. */ | |
5722 | ||
5723 | void * | |
5724 | vuse_eq (ao_ref *, tree vuse1, void *data) | |
5725 | { | |
5726 | tree vuse2 = (tree) data; | |
5727 | if (vuse1 == vuse2) | |
5728 | return data; | |
5729 | ||
5730 | return NULL; | |
5731 | } | |
5732 | ||
5733 | /* Return true if STMT uses a library function and needs to take | |
5734 | address of its inputs. We need to avoid bit-fields in those | |
9f8ba332 JJ |
5735 | cases. Similarly, we need to avoid overlap between destination |
5736 | and source limb arrays. */ | |
a9d6c7fb JJ |
5737 | |
5738 | bool | |
5739 | stmt_needs_operand_addr (gimple *stmt) | |
5740 | { | |
5741 | if (is_gimple_assign (stmt)) | |
5742 | switch (gimple_assign_rhs_code (stmt)) | |
5743 | { | |
5744 | case MULT_EXPR: | |
5745 | case TRUNC_DIV_EXPR: | |
5746 | case TRUNC_MOD_EXPR: | |
5747 | case FLOAT_EXPR: | |
5748 | return true; | |
5749 | default: | |
5750 | break; | |
5751 | } | |
5752 | else if (gimple_call_internal_p (stmt, IFN_MUL_OVERFLOW) | |
5753 | || gimple_call_internal_p (stmt, IFN_UBSAN_CHECK_MUL)) | |
5754 | return true; | |
5755 | return false; | |
5756 | } | |
5757 | ||
5758 | /* Dominator walker used to discover which large/huge _BitInt | |
5759 | loads could be sunk into all their uses. */ | |
5760 | ||
5761 | class bitint_dom_walker : public dom_walker | |
5762 | { | |
5763 | public: | |
5764 | bitint_dom_walker (bitmap names, bitmap loads) | |
5765 | : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {} | |
5766 | ||
5767 | edge before_dom_children (basic_block) final override; | |
5768 | ||
5769 | private: | |
5770 | bitmap m_names, m_loads; | |
5771 | }; | |
5772 | ||
5773 | edge | |
5774 | bitint_dom_walker::before_dom_children (basic_block bb) | |
5775 | { | |
5776 | gphi *phi = get_virtual_phi (bb); | |
5777 | tree vop; | |
5778 | if (phi) | |
5779 | vop = gimple_phi_result (phi); | |
5780 | else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun)) | |
5781 | vop = NULL_TREE; | |
5782 | else | |
5783 | vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux; | |
5784 | ||
5785 | auto_vec<tree, 16> worklist; | |
5786 | for (gimple_stmt_iterator gsi = gsi_start_bb (bb); | |
5787 | !gsi_end_p (gsi); gsi_next (&gsi)) | |
5788 | { | |
5789 | gimple *stmt = gsi_stmt (gsi); | |
5790 | if (is_gimple_debug (stmt)) | |
5791 | continue; | |
5792 | ||
5793 | if (!vop && gimple_vuse (stmt)) | |
5794 | vop = gimple_vuse (stmt); | |
5795 | ||
5796 | tree cvop = vop; | |
5797 | if (gimple_vdef (stmt)) | |
5798 | vop = gimple_vdef (stmt); | |
5799 | ||
5800 | tree lhs = gimple_get_lhs (stmt); | |
5801 | if (lhs | |
5802 | && TREE_CODE (lhs) == SSA_NAME | |
a2d66158 | 5803 | && TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE |
a9d6c7fb JJ |
5804 | && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large |
5805 | && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs))) | |
5806 | /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names, | |
5807 | it means it will be handled in a loop or straight line code | |
5808 | at the location of its (ultimate) immediate use, so for | |
5809 | vop checking purposes check these only at the ultimate | |
5810 | immediate use. */ | |
5811 | continue; | |
5812 | ||
5813 | ssa_op_iter oi; | |
5814 | use_operand_p use_p; | |
5815 | FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE) | |
5816 | { | |
5817 | tree s = USE_FROM_PTR (use_p); | |
5818 | if (TREE_CODE (TREE_TYPE (s)) == BITINT_TYPE | |
5819 | && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large) | |
5820 | worklist.safe_push (s); | |
5821 | } | |
5822 | ||
5823 | bool needs_operand_addr = stmt_needs_operand_addr (stmt); | |
5824 | while (worklist.length () > 0) | |
5825 | { | |
5826 | tree s = worklist.pop (); | |
5827 | ||
5828 | if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s))) | |
5829 | { | |
5830 | gimple *g = SSA_NAME_DEF_STMT (s); | |
5831 | needs_operand_addr |= stmt_needs_operand_addr (g); | |
5832 | FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE) | |
5833 | { | |
5834 | tree s2 = USE_FROM_PTR (use_p); | |
5835 | if (TREE_CODE (TREE_TYPE (s2)) == BITINT_TYPE | |
5836 | && (bitint_precision_kind (TREE_TYPE (s2)) | |
5837 | >= bitint_prec_large)) | |
5838 | worklist.safe_push (s2); | |
5839 | } | |
5840 | continue; | |
5841 | } | |
5842 | if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s) | |
5843 | && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s))) | |
5844 | { | |
5845 | tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s)); | |
5846 | if (TREE_CODE (rhs) == SSA_NAME | |
5847 | && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs))) | |
5848 | s = rhs; | |
5849 | else | |
5850 | continue; | |
5851 | } | |
5852 | else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s))) | |
5853 | continue; | |
5854 | ||
9f8ba332 JJ |
5855 | gimple *g = SSA_NAME_DEF_STMT (s); |
5856 | tree rhs1 = gimple_assign_rhs1 (g); | |
a9d6c7fb JJ |
5857 | if (needs_operand_addr |
5858 | && TREE_CODE (rhs1) == COMPONENT_REF | |
5859 | && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1))) | |
5860 | { | |
5861 | tree fld = TREE_OPERAND (rhs1, 1); | |
5862 | /* For little-endian, we can allow as inputs bit-fields | |
5863 | which start at a limb boundary. */ | |
5864 | if (DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1)) | |
5865 | && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)) | |
5866 | && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) | |
5867 | % limb_prec) == 0) | |
5868 | ; | |
5869 | else | |
5870 | { | |
5871 | bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s)); | |
5872 | continue; | |
5873 | } | |
5874 | } | |
5875 | ||
5876 | ao_ref ref; | |
5877 | ao_ref_init (&ref, rhs1); | |
9f8ba332 | 5878 | tree lvop = gimple_vuse (g); |
a9d6c7fb JJ |
5879 | unsigned limit = 64; |
5880 | tree vuse = cvop; | |
5881 | if (vop != cvop | |
5882 | && is_gimple_assign (stmt) | |
5883 | && gimple_store_p (stmt) | |
9f8ba332 JJ |
5884 | && (needs_operand_addr |
5885 | || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0))) | |
a9d6c7fb JJ |
5886 | vuse = vop; |
5887 | if (vuse != lvop | |
5888 | && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq, | |
5889 | NULL, NULL, limit, lvop) == NULL) | |
5890 | bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s)); | |
5891 | } | |
5892 | } | |
5893 | ||
5894 | bb->aux = (void *) vop; | |
5895 | return NULL; | |
5896 | } | |
5897 | ||
5898 | } | |
5899 | ||
5900 | /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc | |
5901 | build_ssa_conflict_graph. | |
5902 | The differences are: | |
5903 | 1) don't process assignments with large/huge _BitInt lhs not in NAMES | |
5904 | 2) for large/huge _BitInt multiplication/division/modulo process def | |
5905 | only after processing uses rather than before to make uses conflict | |
5906 | with the definition | |
5907 | 3) for large/huge _BitInt uses not in NAMES mark the uses of their | |
5908 | SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into | |
5909 | the final statement. */ | |
5910 | ||
5911 | void | |
5912 | build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live, | |
5913 | ssa_conflicts *graph, bitmap names, | |
5914 | void (*def) (live_track *, tree, | |
5915 | ssa_conflicts *), | |
5916 | void (*use) (live_track *, tree)) | |
5917 | { | |
5918 | bool muldiv_p = false; | |
5919 | tree lhs = NULL_TREE; | |
5920 | if (is_gimple_assign (stmt)) | |
5921 | { | |
5922 | lhs = gimple_assign_lhs (stmt); | |
f92cf8cb | 5923 | if (TREE_CODE (lhs) == SSA_NAME) |
a9d6c7fb | 5924 | { |
f92cf8cb JJ |
5925 | tree type = TREE_TYPE (lhs); |
5926 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
5927 | type = TREE_TYPE (type); | |
5928 | if (TREE_CODE (type) == BITINT_TYPE | |
5929 | && bitint_precision_kind (type) >= bitint_prec_large) | |
a9d6c7fb | 5930 | { |
f92cf8cb JJ |
5931 | if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs))) |
5932 | return; | |
5933 | switch (gimple_assign_rhs_code (stmt)) | |
5934 | { | |
5935 | case MULT_EXPR: | |
5936 | case TRUNC_DIV_EXPR: | |
5937 | case TRUNC_MOD_EXPR: | |
5938 | muldiv_p = true; | |
5939 | default: | |
5940 | break; | |
5941 | } | |
a9d6c7fb JJ |
5942 | } |
5943 | } | |
5944 | } | |
5945 | ||
5946 | ssa_op_iter iter; | |
5947 | tree var; | |
5948 | if (!muldiv_p) | |
5949 | { | |
5950 | /* For stmts with more than one SSA_NAME definition pretend all the | |
5951 | SSA_NAME outputs but the first one are live at this point, so | |
5952 | that conflicts are added in between all those even when they are | |
5953 | actually not really live after the asm, because expansion might | |
5954 | copy those into pseudos after the asm and if multiple outputs | |
5955 | share the same partition, it might overwrite those that should | |
5956 | be live. E.g. | |
5957 | asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a)); | |
5958 | return a; | |
5959 | See PR70593. */ | |
5960 | bool first = true; | |
5961 | FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF) | |
5962 | if (first) | |
5963 | first = false; | |
5964 | else | |
5965 | use (live, var); | |
5966 | ||
5967 | FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF) | |
5968 | def (live, var, graph); | |
5969 | } | |
5970 | ||
5971 | auto_vec<tree, 16> worklist; | |
5972 | FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE) | |
f92cf8cb JJ |
5973 | { |
5974 | tree type = TREE_TYPE (var); | |
5975 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
5976 | type = TREE_TYPE (type); | |
5977 | if (TREE_CODE (type) == BITINT_TYPE | |
5978 | && bitint_precision_kind (type) >= bitint_prec_large) | |
5979 | { | |
5980 | if (bitmap_bit_p (names, SSA_NAME_VERSION (var))) | |
5981 | use (live, var); | |
5982 | else | |
5983 | worklist.safe_push (var); | |
5984 | } | |
5985 | } | |
a9d6c7fb JJ |
5986 | |
5987 | while (worklist.length () > 0) | |
5988 | { | |
5989 | tree s = worklist.pop (); | |
5990 | FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE) | |
f92cf8cb JJ |
5991 | { |
5992 | tree type = TREE_TYPE (var); | |
5993 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
5994 | type = TREE_TYPE (type); | |
5995 | if (TREE_CODE (type) == BITINT_TYPE | |
5996 | && bitint_precision_kind (type) >= bitint_prec_large) | |
5997 | { | |
5998 | if (bitmap_bit_p (names, SSA_NAME_VERSION (var))) | |
5999 | use (live, var); | |
6000 | else | |
6001 | worklist.safe_push (var); | |
6002 | } | |
6003 | } | |
a9d6c7fb JJ |
6004 | } |
6005 | ||
6006 | if (muldiv_p) | |
6007 | def (live, lhs, graph); | |
6008 | } | |
6009 | ||
d50d3d0a JJ |
6010 | /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments, |
6011 | return the largest bitint_prec_kind of them, otherwise return | |
6012 | bitint_prec_small. */ | |
6013 | ||
6014 | static bitint_prec_kind | |
6015 | arith_overflow_arg_kind (gimple *stmt) | |
6016 | { | |
6017 | bitint_prec_kind ret = bitint_prec_small; | |
6018 | if (is_gimple_call (stmt) && gimple_call_internal_p (stmt)) | |
6019 | switch (gimple_call_internal_fn (stmt)) | |
6020 | { | |
6021 | case IFN_ADD_OVERFLOW: | |
6022 | case IFN_SUB_OVERFLOW: | |
6023 | case IFN_MUL_OVERFLOW: | |
6024 | for (int i = 0; i < 2; ++i) | |
6025 | { | |
6026 | tree a = gimple_call_arg (stmt, i); | |
6027 | if (TREE_CODE (a) == INTEGER_CST | |
6028 | && TREE_CODE (TREE_TYPE (a)) == BITINT_TYPE) | |
6029 | { | |
6030 | bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a)); | |
6031 | ret = MAX (ret, kind); | |
6032 | } | |
6033 | } | |
6034 | break; | |
6035 | default: | |
6036 | break; | |
6037 | } | |
6038 | return ret; | |
6039 | } | |
6040 | ||
a9d6c7fb JJ |
6041 | /* Entry point for _BitInt(N) operation lowering during optimization. */ |
6042 | ||
6043 | static unsigned int | |
6044 | gimple_lower_bitint (void) | |
6045 | { | |
6046 | small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0; | |
6047 | limb_prec = 0; | |
6048 | ||
6049 | unsigned int i; | |
6050 | for (i = 0; i < num_ssa_names; ++i) | |
6051 | { | |
6052 | tree s = ssa_name (i); | |
6053 | if (s == NULL) | |
6054 | continue; | |
6055 | tree type = TREE_TYPE (s); | |
6056 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
d50d3d0a JJ |
6057 | { |
6058 | if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s)) | |
6059 | != bitint_prec_small) | |
6060 | break; | |
6061 | type = TREE_TYPE (type); | |
6062 | } | |
a9d6c7fb JJ |
6063 | if (TREE_CODE (type) == BITINT_TYPE |
6064 | && bitint_precision_kind (type) != bitint_prec_small) | |
6065 | break; | |
6066 | /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs | |
6067 | into memory. Such functions could have no large/huge SSA_NAMEs. */ | |
6068 | if (SSA_NAME_IS_VIRTUAL_OPERAND (s)) | |
6069 | { | |
6070 | gimple *g = SSA_NAME_DEF_STMT (s); | |
6071 | if (is_gimple_assign (g) && gimple_store_p (g)) | |
6072 | { | |
6073 | tree t = gimple_assign_rhs1 (g); | |
6074 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6075 | && (bitint_precision_kind (TREE_TYPE (t)) | |
6076 | >= bitint_prec_large)) | |
6077 | break; | |
6078 | } | |
6079 | } | |
31669ec1 JJ |
6080 | /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs |
6081 | to floating point types need to be rewritten. */ | |
6082 | else if (SCALAR_FLOAT_TYPE_P (type)) | |
6083 | { | |
6084 | gimple *g = SSA_NAME_DEF_STMT (s); | |
6085 | if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR) | |
6086 | { | |
6087 | tree t = gimple_assign_rhs1 (g); | |
6088 | if (TREE_CODE (t) == INTEGER_CST | |
6089 | && TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6090 | && (bitint_precision_kind (TREE_TYPE (t)) | |
6091 | != bitint_prec_small)) | |
6092 | break; | |
6093 | } | |
6094 | } | |
a9d6c7fb JJ |
6095 | } |
6096 | if (i == num_ssa_names) | |
6097 | return 0; | |
6098 | ||
6099 | basic_block bb; | |
6100 | auto_vec<gimple *, 4> switch_statements; | |
6101 | FOR_EACH_BB_FN (bb, cfun) | |
6102 | { | |
6103 | if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb))) | |
6104 | { | |
6105 | tree idx = gimple_switch_index (swtch); | |
6106 | if (TREE_CODE (TREE_TYPE (idx)) != BITINT_TYPE | |
6107 | || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large) | |
6108 | continue; | |
6109 | ||
6110 | if (optimize) | |
6111 | group_case_labels_stmt (swtch); | |
dede174f JJ |
6112 | if (gimple_switch_num_labels (swtch) == 1) |
6113 | { | |
6114 | single_succ_edge (bb)->flags |= EDGE_FALLTHRU; | |
6115 | gimple_stmt_iterator gsi = gsi_for_stmt (swtch); | |
6116 | gsi_remove (&gsi, true); | |
6117 | } | |
6118 | else | |
6119 | switch_statements.safe_push (swtch); | |
a9d6c7fb JJ |
6120 | } |
6121 | } | |
6122 | ||
6123 | if (!switch_statements.is_empty ()) | |
6124 | { | |
6125 | bool expanded = false; | |
6126 | gimple *stmt; | |
6127 | unsigned int j; | |
6128 | i = 0; | |
6129 | FOR_EACH_VEC_ELT (switch_statements, j, stmt) | |
6130 | { | |
6131 | gswitch *swtch = as_a<gswitch *> (stmt); | |
6132 | tree_switch_conversion::switch_decision_tree dt (swtch); | |
6133 | expanded |= dt.analyze_switch_statement (); | |
6134 | } | |
6135 | ||
6136 | if (expanded) | |
6137 | { | |
6138 | free_dominance_info (CDI_DOMINATORS); | |
6139 | free_dominance_info (CDI_POST_DOMINATORS); | |
6140 | mark_virtual_operands_for_renaming (cfun); | |
6141 | cleanup_tree_cfg (TODO_update_ssa); | |
6142 | } | |
6143 | } | |
6144 | ||
6145 | struct bitint_large_huge large_huge; | |
6146 | bool has_large_huge_parm_result = false; | |
6147 | bool has_large_huge = false; | |
6148 | unsigned int ret = 0, first_large_huge = ~0U; | |
6149 | bool edge_insertions = false; | |
6150 | for (; i < num_ssa_names; ++i) | |
6151 | { | |
6152 | tree s = ssa_name (i); | |
6153 | if (s == NULL) | |
6154 | continue; | |
6155 | tree type = TREE_TYPE (s); | |
6156 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
d50d3d0a JJ |
6157 | { |
6158 | if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s)) | |
6159 | >= bitint_prec_large) | |
6160 | has_large_huge = true; | |
6161 | type = TREE_TYPE (type); | |
6162 | } | |
a9d6c7fb JJ |
6163 | if (TREE_CODE (type) == BITINT_TYPE |
6164 | && bitint_precision_kind (type) >= bitint_prec_large) | |
6165 | { | |
6166 | if (first_large_huge == ~0U) | |
6167 | first_large_huge = i; | |
6168 | gimple *stmt = SSA_NAME_DEF_STMT (s), *g; | |
6169 | gimple_stmt_iterator gsi; | |
6170 | tree_code rhs_code; | |
6171 | /* Unoptimize certain constructs to simpler alternatives to | |
6172 | avoid having to lower all of them. */ | |
f5198f02 | 6173 | if (is_gimple_assign (stmt) && gimple_bb (stmt)) |
a9d6c7fb JJ |
6174 | switch (rhs_code = gimple_assign_rhs_code (stmt)) |
6175 | { | |
6176 | default: | |
6177 | break; | |
baa40971 JJ |
6178 | case MULT_EXPR: |
6179 | case TRUNC_DIV_EXPR: | |
6180 | case TRUNC_MOD_EXPR: | |
6181 | if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)) | |
6182 | { | |
6183 | location_t loc = gimple_location (stmt); | |
6184 | gsi = gsi_for_stmt (stmt); | |
6185 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
6186 | tree rhs2 = gimple_assign_rhs2 (stmt); | |
6187 | /* For multiplication and division with (ab) | |
6188 | lhs and one or both operands force the operands | |
6189 | into new SSA_NAMEs to avoid coalescing failures. */ | |
6190 | if (TREE_CODE (rhs1) == SSA_NAME | |
6191 | && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)) | |
6192 | { | |
6193 | first_large_huge = 0; | |
6194 | tree t = make_ssa_name (TREE_TYPE (rhs1)); | |
6195 | g = gimple_build_assign (t, SSA_NAME, rhs1); | |
6196 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6197 | gimple_set_location (g, loc); | |
6198 | gimple_assign_set_rhs1 (stmt, t); | |
6199 | if (rhs1 == rhs2) | |
6200 | { | |
6201 | gimple_assign_set_rhs2 (stmt, t); | |
6202 | rhs2 = t; | |
6203 | } | |
6204 | update_stmt (stmt); | |
6205 | } | |
6206 | if (TREE_CODE (rhs2) == SSA_NAME | |
6207 | && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2)) | |
6208 | { | |
6209 | first_large_huge = 0; | |
6210 | tree t = make_ssa_name (TREE_TYPE (rhs2)); | |
6211 | g = gimple_build_assign (t, SSA_NAME, rhs2); | |
6212 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6213 | gimple_set_location (g, loc); | |
6214 | gimple_assign_set_rhs2 (stmt, t); | |
6215 | update_stmt (stmt); | |
6216 | } | |
6217 | } | |
6218 | break; | |
a9d6c7fb JJ |
6219 | case LROTATE_EXPR: |
6220 | case RROTATE_EXPR: | |
6221 | { | |
6222 | first_large_huge = 0; | |
6223 | location_t loc = gimple_location (stmt); | |
6224 | gsi = gsi_for_stmt (stmt); | |
6225 | tree rhs1 = gimple_assign_rhs1 (stmt); | |
6226 | tree type = TREE_TYPE (rhs1); | |
6227 | tree n = gimple_assign_rhs2 (stmt), m; | |
6228 | tree p = build_int_cst (TREE_TYPE (n), | |
6229 | TYPE_PRECISION (type)); | |
6230 | if (TREE_CODE (n) == INTEGER_CST) | |
6231 | m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n); | |
6232 | else | |
6233 | { | |
6234 | m = make_ssa_name (TREE_TYPE (n)); | |
6235 | g = gimple_build_assign (m, MINUS_EXPR, p, n); | |
6236 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6237 | gimple_set_location (g, loc); | |
6238 | } | |
6239 | if (!TYPE_UNSIGNED (type)) | |
6240 | { | |
6241 | tree utype = build_bitint_type (TYPE_PRECISION (type), | |
6242 | 1); | |
6243 | if (TREE_CODE (rhs1) == INTEGER_CST) | |
6244 | rhs1 = fold_convert (utype, rhs1); | |
6245 | else | |
6246 | { | |
6247 | tree t = make_ssa_name (type); | |
6248 | g = gimple_build_assign (t, NOP_EXPR, rhs1); | |
6249 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6250 | gimple_set_location (g, loc); | |
6251 | } | |
6252 | } | |
6253 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)), | |
6254 | rhs_code == LROTATE_EXPR | |
6255 | ? LSHIFT_EXPR : RSHIFT_EXPR, | |
6256 | rhs1, n); | |
6257 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6258 | gimple_set_location (g, loc); | |
6259 | tree op1 = gimple_assign_lhs (g); | |
6260 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)), | |
6261 | rhs_code == LROTATE_EXPR | |
6262 | ? RSHIFT_EXPR : LSHIFT_EXPR, | |
6263 | rhs1, m); | |
6264 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6265 | gimple_set_location (g, loc); | |
6266 | tree op2 = gimple_assign_lhs (g); | |
6267 | tree lhs = gimple_assign_lhs (stmt); | |
6268 | if (!TYPE_UNSIGNED (type)) | |
6269 | { | |
6270 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)), | |
6271 | BIT_IOR_EXPR, op1, op2); | |
6272 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6273 | gimple_set_location (g, loc); | |
6274 | g = gimple_build_assign (lhs, NOP_EXPR, | |
6275 | gimple_assign_lhs (g)); | |
6276 | } | |
6277 | else | |
6278 | g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2); | |
6279 | gsi_replace (&gsi, g, true); | |
6280 | gimple_set_location (g, loc); | |
6281 | } | |
6282 | break; | |
6283 | case ABS_EXPR: | |
6284 | case ABSU_EXPR: | |
6285 | case MIN_EXPR: | |
6286 | case MAX_EXPR: | |
6287 | case COND_EXPR: | |
6288 | first_large_huge = 0; | |
6289 | gsi = gsi_for_stmt (stmt); | |
6290 | tree lhs = gimple_assign_lhs (stmt); | |
6291 | tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE; | |
6292 | location_t loc = gimple_location (stmt); | |
6293 | if (rhs_code == ABS_EXPR) | |
6294 | g = gimple_build_cond (LT_EXPR, rhs1, | |
6295 | build_zero_cst (TREE_TYPE (rhs1)), | |
6296 | NULL_TREE, NULL_TREE); | |
6297 | else if (rhs_code == ABSU_EXPR) | |
6298 | { | |
6299 | rhs2 = make_ssa_name (TREE_TYPE (lhs)); | |
6300 | g = gimple_build_assign (rhs2, NOP_EXPR, rhs1); | |
6301 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6302 | gimple_set_location (g, loc); | |
6303 | g = gimple_build_cond (LT_EXPR, rhs1, | |
6304 | build_zero_cst (TREE_TYPE (rhs1)), | |
6305 | NULL_TREE, NULL_TREE); | |
6306 | rhs1 = rhs2; | |
6307 | } | |
6308 | else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR) | |
6309 | { | |
6310 | rhs2 = gimple_assign_rhs2 (stmt); | |
6311 | if (TREE_CODE (rhs1) == INTEGER_CST) | |
6312 | std::swap (rhs1, rhs2); | |
6313 | g = gimple_build_cond (LT_EXPR, rhs1, rhs2, | |
6314 | NULL_TREE, NULL_TREE); | |
6315 | if (rhs_code == MAX_EXPR) | |
6316 | std::swap (rhs1, rhs2); | |
6317 | } | |
6318 | else | |
6319 | { | |
6320 | g = gimple_build_cond (NE_EXPR, rhs1, | |
6321 | build_zero_cst (TREE_TYPE (rhs1)), | |
6322 | NULL_TREE, NULL_TREE); | |
6323 | rhs1 = gimple_assign_rhs2 (stmt); | |
6324 | rhs2 = gimple_assign_rhs3 (stmt); | |
6325 | } | |
6326 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6327 | gimple_set_location (g, loc); | |
6328 | edge e1 = split_block (gsi_bb (gsi), g); | |
6329 | edge e2 = split_block (e1->dest, (gimple *) NULL); | |
6330 | edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE); | |
6331 | e3->probability = profile_probability::even (); | |
6332 | e1->flags = EDGE_TRUE_VALUE; | |
6333 | e1->probability = e3->probability.invert (); | |
6334 | if (dom_info_available_p (CDI_DOMINATORS)) | |
6335 | set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src); | |
6336 | if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR) | |
6337 | { | |
6338 | gsi = gsi_after_labels (e1->dest); | |
6339 | g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)), | |
6340 | NEGATE_EXPR, rhs1); | |
6341 | gsi_insert_before (&gsi, g, GSI_SAME_STMT); | |
6342 | gimple_set_location (g, loc); | |
6343 | rhs2 = gimple_assign_lhs (g); | |
6344 | std::swap (rhs1, rhs2); | |
6345 | } | |
6346 | gsi = gsi_for_stmt (stmt); | |
6347 | gsi_remove (&gsi, true); | |
6348 | gphi *phi = create_phi_node (lhs, e2->dest); | |
6349 | add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION); | |
6350 | add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION); | |
6351 | break; | |
6352 | } | |
6353 | } | |
6354 | /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs | |
6355 | into memory. Such functions could have no large/huge SSA_NAMEs. */ | |
6356 | else if (SSA_NAME_IS_VIRTUAL_OPERAND (s)) | |
6357 | { | |
6358 | gimple *g = SSA_NAME_DEF_STMT (s); | |
6359 | if (is_gimple_assign (g) && gimple_store_p (g)) | |
6360 | { | |
6361 | tree t = gimple_assign_rhs1 (g); | |
6362 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6363 | && (bitint_precision_kind (TREE_TYPE (t)) | |
6364 | >= bitint_prec_large)) | |
6365 | has_large_huge = true; | |
6366 | } | |
6367 | } | |
31669ec1 JJ |
6368 | /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs |
6369 | to floating point types need to be rewritten. */ | |
6370 | else if (SCALAR_FLOAT_TYPE_P (type)) | |
6371 | { | |
6372 | gimple *g = SSA_NAME_DEF_STMT (s); | |
6373 | if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR) | |
6374 | { | |
6375 | tree t = gimple_assign_rhs1 (g); | |
6376 | if (TREE_CODE (t) == INTEGER_CST | |
6377 | && TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6378 | && (bitint_precision_kind (TREE_TYPE (t)) | |
6379 | >= bitint_prec_large)) | |
6380 | has_large_huge = true; | |
6381 | } | |
6382 | } | |
a9d6c7fb JJ |
6383 | } |
6384 | for (i = first_large_huge; i < num_ssa_names; ++i) | |
6385 | { | |
6386 | tree s = ssa_name (i); | |
6387 | if (s == NULL) | |
6388 | continue; | |
6389 | tree type = TREE_TYPE (s); | |
6390 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
6391 | type = TREE_TYPE (type); | |
6392 | if (TREE_CODE (type) == BITINT_TYPE | |
6393 | && bitint_precision_kind (type) >= bitint_prec_large) | |
6394 | { | |
6395 | use_operand_p use_p; | |
6396 | gimple *use_stmt; | |
6397 | has_large_huge = true; | |
6398 | if (optimize | |
6399 | && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s))) | |
6400 | continue; | |
6401 | /* Ignore large/huge _BitInt SSA_NAMEs which have single use in | |
6402 | the same bb and could be handled in the same loop with the | |
6403 | immediate use. */ | |
6404 | if (optimize | |
6405 | && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s) | |
6406 | && single_imm_use (s, &use_p, &use_stmt) | |
6407 | && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt)) | |
6408 | { | |
6409 | if (mergeable_op (SSA_NAME_DEF_STMT (s))) | |
6410 | { | |
6411 | if (mergeable_op (use_stmt)) | |
6412 | continue; | |
6413 | tree_code cmp_code = comparison_op (use_stmt, NULL, NULL); | |
6414 | if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR) | |
6415 | continue; | |
6416 | if (gimple_assign_cast_p (use_stmt)) | |
6417 | { | |
6418 | tree lhs = gimple_assign_lhs (use_stmt); | |
c9bdcb0c JJ |
6419 | if (INTEGRAL_TYPE_P (TREE_TYPE (lhs)) |
6420 | /* Don't merge with VIEW_CONVERT_EXPRs to | |
6421 | huge INTEGER_TYPEs used sometimes in memcpy | |
6422 | expansion. */ | |
6423 | && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE | |
6424 | || (TYPE_PRECISION (TREE_TYPE (lhs)) | |
6425 | <= MAX_FIXED_MODE_SIZE))) | |
a9d6c7fb JJ |
6426 | continue; |
6427 | } | |
6428 | else if (gimple_store_p (use_stmt) | |
6429 | && is_gimple_assign (use_stmt) | |
6430 | && !gimple_has_volatile_ops (use_stmt) | |
6431 | && !stmt_ends_bb_p (use_stmt)) | |
6432 | continue; | |
6433 | } | |
6434 | if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s))) | |
6435 | { | |
6436 | tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s)); | |
5e7a176e JJ |
6437 | if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR) |
6438 | { | |
6439 | rhs1 = TREE_OPERAND (rhs1, 0); | |
6440 | if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1)) | |
6441 | && !POINTER_TYPE_P (TREE_TYPE (rhs1)) | |
6442 | && gimple_store_p (use_stmt)) | |
6443 | continue; | |
6444 | } | |
a9d6c7fb JJ |
6445 | if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)) |
6446 | && ((is_gimple_assign (use_stmt) | |
6447 | && (gimple_assign_rhs_code (use_stmt) | |
6448 | != COMPLEX_EXPR)) | |
6449 | || gimple_code (use_stmt) == GIMPLE_COND) | |
6450 | && (!gimple_store_p (use_stmt) | |
6451 | || (is_gimple_assign (use_stmt) | |
6452 | && !gimple_has_volatile_ops (use_stmt) | |
6453 | && !stmt_ends_bb_p (use_stmt))) | |
6454 | && (TREE_CODE (rhs1) != SSA_NAME | |
6455 | || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))) | |
6456 | { | |
3d1bdbf6 JJ |
6457 | if (is_gimple_assign (use_stmt)) |
6458 | switch (gimple_assign_rhs_code (use_stmt)) | |
6459 | { | |
3d1bdbf6 JJ |
6460 | case TRUNC_DIV_EXPR: |
6461 | case TRUNC_MOD_EXPR: | |
6462 | case FLOAT_EXPR: | |
a12b0e93 JJ |
6463 | /* For division, modulo and casts to floating |
6464 | point, avoid representing unsigned operands | |
6465 | using negative prec if they were sign-extended | |
6466 | from narrower precision. */ | |
6467 | if (TYPE_UNSIGNED (TREE_TYPE (s)) | |
6468 | && !TYPE_UNSIGNED (TREE_TYPE (rhs1)) | |
6469 | && (TYPE_PRECISION (TREE_TYPE (s)) | |
6470 | > TYPE_PRECISION (TREE_TYPE (rhs1)))) | |
6471 | goto force_name; | |
6472 | /* FALLTHRU */ | |
6473 | case MULT_EXPR: | |
6474 | if (TREE_CODE (TREE_TYPE (rhs1)) != BITINT_TYPE | |
6475 | || (bitint_precision_kind (TREE_TYPE (rhs1)) | |
6476 | < bitint_prec_large)) | |
6477 | continue; | |
3d1bdbf6 JJ |
6478 | /* Uses which use handle_operand_addr can't |
6479 | deal with nested casts. */ | |
6480 | if (TREE_CODE (rhs1) == SSA_NAME | |
6481 | && gimple_assign_cast_p | |
6482 | (SSA_NAME_DEF_STMT (rhs1)) | |
6483 | && has_single_use (rhs1) | |
6484 | && (gimple_bb (SSA_NAME_DEF_STMT (rhs1)) | |
6485 | == gimple_bb (SSA_NAME_DEF_STMT (s)))) | |
6486 | goto force_name; | |
6487 | break; | |
c9bdcb0c | 6488 | case VIEW_CONVERT_EXPR: |
c9bdcb0c JJ |
6489 | { |
6490 | tree lhs = gimple_assign_lhs (use_stmt); | |
5e7a176e JJ |
6491 | /* Don't merge with VIEW_CONVERT_EXPRs to |
6492 | non-integral types. */ | |
6493 | if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))) | |
6494 | goto force_name; | |
6495 | /* Don't merge with VIEW_CONVERT_EXPRs to | |
6496 | huge INTEGER_TYPEs used sometimes in memcpy | |
6497 | expansion. */ | |
c9bdcb0c JJ |
6498 | if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE |
6499 | && (TYPE_PRECISION (TREE_TYPE (lhs)) | |
6500 | > MAX_FIXED_MODE_SIZE)) | |
6501 | goto force_name; | |
6502 | } | |
6503 | break; | |
3d1bdbf6 JJ |
6504 | default: |
6505 | break; | |
6506 | } | |
a12b0e93 JJ |
6507 | if (TREE_CODE (TREE_TYPE (rhs1)) != BITINT_TYPE |
6508 | || (bitint_precision_kind (TREE_TYPE (rhs1)) | |
6509 | < bitint_prec_large)) | |
6510 | continue; | |
f32e49ad JJ |
6511 | if ((TYPE_PRECISION (TREE_TYPE (rhs1)) |
6512 | >= TYPE_PRECISION (TREE_TYPE (s))) | |
6513 | && mergeable_op (use_stmt)) | |
a9d6c7fb JJ |
6514 | continue; |
6515 | /* Prevent merging a widening non-mergeable cast | |
6516 | on result of some narrower mergeable op | |
6517 | together with later mergeable operations. E.g. | |
6518 | result of _BitInt(223) addition shouldn't be | |
6519 | sign-extended to _BitInt(513) and have another | |
6520 | _BitInt(513) added to it, as handle_plus_minus | |
6521 | with its PHI node handling inside of handle_cast | |
6522 | will not work correctly. An exception is if | |
6523 | use_stmt is a store, this is handled directly | |
6524 | in lower_mergeable_stmt. */ | |
6525 | if (TREE_CODE (rhs1) != SSA_NAME | |
6526 | || !has_single_use (rhs1) | |
6527 | || (gimple_bb (SSA_NAME_DEF_STMT (rhs1)) | |
6528 | != gimple_bb (SSA_NAME_DEF_STMT (s))) | |
6529 | || !mergeable_op (SSA_NAME_DEF_STMT (rhs1)) | |
6530 | || gimple_store_p (use_stmt)) | |
6531 | continue; | |
f32e49ad JJ |
6532 | if ((TYPE_PRECISION (TREE_TYPE (rhs1)) |
6533 | < TYPE_PRECISION (TREE_TYPE (s))) | |
6534 | && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1))) | |
a9d6c7fb JJ |
6535 | { |
6536 | /* Another exception is if the widening cast is | |
6537 | from mergeable same precision cast from something | |
6538 | not mergeable. */ | |
6539 | tree rhs2 | |
6540 | = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1)); | |
6541 | if (TREE_CODE (TREE_TYPE (rhs2)) == BITINT_TYPE | |
6542 | && (TYPE_PRECISION (TREE_TYPE (rhs1)) | |
6543 | == TYPE_PRECISION (TREE_TYPE (rhs2)))) | |
6544 | { | |
6545 | if (TREE_CODE (rhs2) != SSA_NAME | |
6546 | || !has_single_use (rhs2) | |
6547 | || (gimple_bb (SSA_NAME_DEF_STMT (rhs2)) | |
6548 | != gimple_bb (SSA_NAME_DEF_STMT (s))) | |
6549 | || !mergeable_op (SSA_NAME_DEF_STMT (rhs2))) | |
6550 | continue; | |
6551 | } | |
6552 | } | |
6553 | } | |
6554 | } | |
6555 | if (is_gimple_assign (SSA_NAME_DEF_STMT (s))) | |
6556 | switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s))) | |
6557 | { | |
6558 | case IMAGPART_EXPR: | |
6559 | { | |
6560 | tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s)); | |
6561 | rhs1 = TREE_OPERAND (rhs1, 0); | |
6562 | if (TREE_CODE (rhs1) == SSA_NAME) | |
6563 | { | |
6564 | gimple *g = SSA_NAME_DEF_STMT (rhs1); | |
6565 | if (optimizable_arith_overflow (g)) | |
6566 | continue; | |
6567 | } | |
6568 | } | |
6569 | /* FALLTHRU */ | |
6570 | case LSHIFT_EXPR: | |
6571 | case RSHIFT_EXPR: | |
6572 | case MULT_EXPR: | |
6573 | case TRUNC_DIV_EXPR: | |
6574 | case TRUNC_MOD_EXPR: | |
6575 | case FIX_TRUNC_EXPR: | |
6576 | case REALPART_EXPR: | |
6577 | if (gimple_store_p (use_stmt) | |
6578 | && is_gimple_assign (use_stmt) | |
6579 | && !gimple_has_volatile_ops (use_stmt) | |
6580 | && !stmt_ends_bb_p (use_stmt)) | |
6581 | { | |
6582 | tree lhs = gimple_assign_lhs (use_stmt); | |
6583 | /* As multiply/division passes address of the lhs | |
6584 | to library function and that assumes it can extend | |
6585 | it to whole number of limbs, avoid merging those | |
6586 | with bit-field stores. Don't allow it for | |
6587 | shifts etc. either, so that the bit-field store | |
6588 | handling doesn't have to be done everywhere. */ | |
6589 | if (TREE_CODE (lhs) == COMPONENT_REF | |
6590 | && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1))) | |
6591 | break; | |
6592 | continue; | |
6593 | } | |
6594 | break; | |
6595 | default: | |
6596 | break; | |
6597 | } | |
6598 | } | |
6599 | ||
6600 | /* Also ignore uninitialized uses. */ | |
6601 | if (SSA_NAME_IS_DEFAULT_DEF (s) | |
6602 | && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s)))) | |
6603 | continue; | |
6604 | ||
3d1bdbf6 | 6605 | force_name: |
a9d6c7fb JJ |
6606 | if (!large_huge.m_names) |
6607 | large_huge.m_names = BITMAP_ALLOC (NULL); | |
6608 | bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s)); | |
6609 | if (has_single_use (s)) | |
6610 | { | |
6611 | if (!large_huge.m_single_use_names) | |
6612 | large_huge.m_single_use_names = BITMAP_ALLOC (NULL); | |
6613 | bitmap_set_bit (large_huge.m_single_use_names, | |
6614 | SSA_NAME_VERSION (s)); | |
6615 | } | |
6616 | if (SSA_NAME_VAR (s) | |
6617 | && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL | |
6618 | && SSA_NAME_IS_DEFAULT_DEF (s)) | |
6619 | || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL)) | |
6620 | has_large_huge_parm_result = true; | |
6621 | if (optimize | |
6622 | && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s) | |
6623 | && gimple_assign_load_p (SSA_NAME_DEF_STMT (s)) | |
6624 | && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s)) | |
6625 | && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s))) | |
6626 | { | |
6627 | use_operand_p use_p; | |
6628 | imm_use_iterator iter; | |
6629 | bool optimizable_load = true; | |
6630 | FOR_EACH_IMM_USE_FAST (use_p, iter, s) | |
6631 | { | |
6632 | gimple *use_stmt = USE_STMT (use_p); | |
6633 | if (is_gimple_debug (use_stmt)) | |
6634 | continue; | |
6635 | if (gimple_code (use_stmt) == GIMPLE_PHI | |
ee2aa93e JJ |
6636 | || is_gimple_call (use_stmt) |
6637 | || gimple_code (use_stmt) == GIMPLE_ASM) | |
a9d6c7fb JJ |
6638 | { |
6639 | optimizable_load = false; | |
6640 | break; | |
6641 | } | |
6642 | } | |
6643 | ||
6644 | ssa_op_iter oi; | |
6645 | FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s), | |
6646 | oi, SSA_OP_USE) | |
6647 | { | |
6648 | tree s2 = USE_FROM_PTR (use_p); | |
6649 | if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2)) | |
6650 | { | |
6651 | optimizable_load = false; | |
6652 | break; | |
6653 | } | |
6654 | } | |
6655 | ||
6656 | if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s))) | |
6657 | { | |
6658 | if (!large_huge.m_loads) | |
6659 | large_huge.m_loads = BITMAP_ALLOC (NULL); | |
6660 | bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s)); | |
6661 | } | |
6662 | } | |
6663 | } | |
6664 | /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs | |
6665 | into memory. Such functions could have no large/huge SSA_NAMEs. */ | |
6666 | else if (SSA_NAME_IS_VIRTUAL_OPERAND (s)) | |
6667 | { | |
6668 | gimple *g = SSA_NAME_DEF_STMT (s); | |
6669 | if (is_gimple_assign (g) && gimple_store_p (g)) | |
6670 | { | |
6671 | tree t = gimple_assign_rhs1 (g); | |
6672 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6673 | && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large) | |
6674 | has_large_huge = true; | |
6675 | } | |
6676 | } | |
6677 | } | |
6678 | ||
6679 | if (large_huge.m_names || has_large_huge) | |
6680 | { | |
6681 | ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg; | |
6682 | calculate_dominance_info (CDI_DOMINATORS); | |
6683 | if (optimize) | |
6684 | enable_ranger (cfun); | |
6685 | if (large_huge.m_loads) | |
6686 | { | |
6687 | basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun); | |
6688 | entry->aux = NULL; | |
6689 | bitint_dom_walker (large_huge.m_names, | |
6690 | large_huge.m_loads).walk (entry); | |
6691 | bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads); | |
6692 | clear_aux_for_blocks (); | |
6693 | BITMAP_FREE (large_huge.m_loads); | |
6694 | } | |
6695 | large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1); | |
6696 | large_huge.m_limb_size | |
6697 | = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type)); | |
6698 | } | |
6699 | if (large_huge.m_names) | |
6700 | { | |
6701 | large_huge.m_map | |
6702 | = init_var_map (num_ssa_names, NULL, large_huge.m_names); | |
6703 | coalesce_ssa_name (large_huge.m_map); | |
6704 | partition_view_normal (large_huge.m_map); | |
6705 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
6706 | { | |
6707 | fprintf (dump_file, "After Coalescing:\n"); | |
6708 | dump_var_map (dump_file, large_huge.m_map); | |
6709 | } | |
6710 | large_huge.m_vars | |
6711 | = XCNEWVEC (tree, num_var_partitions (large_huge.m_map)); | |
6712 | bitmap_iterator bi; | |
6713 | if (has_large_huge_parm_result) | |
6714 | EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi) | |
6715 | { | |
6716 | tree s = ssa_name (i); | |
6717 | if (SSA_NAME_VAR (s) | |
6718 | && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL | |
6719 | && SSA_NAME_IS_DEFAULT_DEF (s)) | |
6720 | || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL)) | |
6721 | { | |
6722 | int p = var_to_partition (large_huge.m_map, s); | |
6723 | if (large_huge.m_vars[p] == NULL_TREE) | |
6724 | { | |
6725 | large_huge.m_vars[p] = SSA_NAME_VAR (s); | |
6726 | mark_addressable (SSA_NAME_VAR (s)); | |
6727 | } | |
6728 | } | |
6729 | } | |
6730 | tree atype = NULL_TREE; | |
675e5229 JJ |
6731 | if (dump_file && (dump_flags & TDF_DETAILS)) |
6732 | fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n"); | |
a9d6c7fb JJ |
6733 | EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi) |
6734 | { | |
6735 | tree s = ssa_name (i); | |
6736 | int p = var_to_partition (large_huge.m_map, s); | |
675e5229 JJ |
6737 | if (large_huge.m_vars[p] == NULL_TREE) |
6738 | { | |
6739 | if (atype == NULL_TREE | |
6740 | || !tree_int_cst_equal (TYPE_SIZE (atype), | |
6741 | TYPE_SIZE (TREE_TYPE (s)))) | |
6742 | { | |
6743 | unsigned HOST_WIDE_INT nelts | |
6744 | = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec; | |
6745 | atype = build_array_type_nelts (large_huge.m_limb_type, | |
6746 | nelts); | |
6747 | } | |
6748 | large_huge.m_vars[p] = create_tmp_var (atype, "bitint"); | |
6749 | mark_addressable (large_huge.m_vars[p]); | |
6750 | } | |
6751 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
a9d6c7fb | 6752 | { |
675e5229 JJ |
6753 | print_generic_expr (dump_file, s, TDF_SLIM); |
6754 | fprintf (dump_file, " -> "); | |
6755 | print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM); | |
6756 | fprintf (dump_file, "\n"); | |
a9d6c7fb | 6757 | } |
a9d6c7fb JJ |
6758 | } |
6759 | } | |
6760 | ||
6761 | FOR_EACH_BB_REVERSE_FN (bb, cfun) | |
6762 | { | |
6763 | gimple_stmt_iterator prev; | |
6764 | for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi); | |
6765 | gsi = prev) | |
6766 | { | |
6767 | prev = gsi; | |
6768 | gsi_prev (&prev); | |
6769 | ssa_op_iter iter; | |
6770 | gimple *stmt = gsi_stmt (gsi); | |
6771 | if (is_gimple_debug (stmt)) | |
6772 | continue; | |
6773 | bitint_prec_kind kind = bitint_prec_small; | |
6774 | tree t; | |
6775 | FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS) | |
6776 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE) | |
6777 | { | |
6778 | bitint_prec_kind this_kind | |
6779 | = bitint_precision_kind (TREE_TYPE (t)); | |
d50d3d0a | 6780 | kind = MAX (kind, this_kind); |
a9d6c7fb JJ |
6781 | } |
6782 | if (is_gimple_assign (stmt) && gimple_store_p (stmt)) | |
6783 | { | |
6784 | t = gimple_assign_rhs1 (stmt); | |
6785 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE) | |
6786 | { | |
6787 | bitint_prec_kind this_kind | |
6788 | = bitint_precision_kind (TREE_TYPE (t)); | |
d50d3d0a | 6789 | kind = MAX (kind, this_kind); |
a9d6c7fb JJ |
6790 | } |
6791 | } | |
31669ec1 JJ |
6792 | if (is_gimple_assign (stmt) |
6793 | && gimple_assign_rhs_code (stmt) == FLOAT_EXPR) | |
6794 | { | |
6795 | t = gimple_assign_rhs1 (stmt); | |
6796 | if (TREE_CODE (TREE_TYPE (t)) == BITINT_TYPE | |
6797 | && TREE_CODE (t) == INTEGER_CST) | |
6798 | { | |
6799 | bitint_prec_kind this_kind | |
6800 | = bitint_precision_kind (TREE_TYPE (t)); | |
d50d3d0a | 6801 | kind = MAX (kind, this_kind); |
31669ec1 JJ |
6802 | } |
6803 | } | |
a9d6c7fb JJ |
6804 | if (is_gimple_call (stmt)) |
6805 | { | |
6806 | t = gimple_call_lhs (stmt); | |
d50d3d0a | 6807 | if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE) |
a9d6c7fb | 6808 | { |
d50d3d0a JJ |
6809 | bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt); |
6810 | kind = MAX (kind, this_kind); | |
6811 | if (TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == BITINT_TYPE) | |
6812 | { | |
6813 | this_kind | |
6814 | = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t))); | |
6815 | kind = MAX (kind, this_kind); | |
6816 | } | |
a9d6c7fb JJ |
6817 | } |
6818 | } | |
6819 | if (kind == bitint_prec_small) | |
6820 | continue; | |
6821 | switch (gimple_code (stmt)) | |
6822 | { | |
6823 | case GIMPLE_CALL: | |
6824 | /* For now. We'll need to handle some internal functions and | |
6825 | perhaps some builtins. */ | |
6826 | if (kind == bitint_prec_middle) | |
6827 | continue; | |
6828 | break; | |
6829 | case GIMPLE_ASM: | |
6830 | if (kind == bitint_prec_middle) | |
6831 | continue; | |
6832 | break; | |
6833 | case GIMPLE_RETURN: | |
6834 | continue; | |
6835 | case GIMPLE_ASSIGN: | |
6836 | if (gimple_clobber_p (stmt)) | |
6837 | continue; | |
6838 | if (kind >= bitint_prec_large) | |
6839 | break; | |
6840 | if (gimple_assign_single_p (stmt)) | |
6841 | /* No need to lower copies, loads or stores. */ | |
6842 | continue; | |
6843 | if (gimple_assign_cast_p (stmt)) | |
6844 | { | |
6845 | tree lhs = gimple_assign_lhs (stmt); | |
6846 | tree rhs = gimple_assign_rhs1 (stmt); | |
6847 | if (INTEGRAL_TYPE_P (TREE_TYPE (lhs)) | |
6848 | && INTEGRAL_TYPE_P (TREE_TYPE (rhs)) | |
6849 | && (TYPE_PRECISION (TREE_TYPE (lhs)) | |
6850 | == TYPE_PRECISION (TREE_TYPE (rhs)))) | |
6851 | /* No need to lower casts to same precision. */ | |
6852 | continue; | |
6853 | } | |
6854 | break; | |
6855 | default: | |
6856 | break; | |
6857 | } | |
6858 | ||
6859 | if (kind == bitint_prec_middle) | |
6860 | { | |
6861 | tree type = NULL_TREE; | |
6862 | /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs | |
6863 | with the same precision and back. */ | |
a9d6c7fb | 6864 | unsigned int nops = gimple_num_ops (stmt); |
1a84af19 JJ |
6865 | for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0; |
6866 | i < nops; ++i) | |
a9d6c7fb JJ |
6867 | if (tree op = gimple_op (stmt, i)) |
6868 | { | |
6869 | tree nop = maybe_cast_middle_bitint (&gsi, op, type); | |
6870 | if (nop != op) | |
6871 | gimple_set_op (stmt, i, nop); | |
6872 | else if (COMPARISON_CLASS_P (op)) | |
6873 | { | |
6874 | TREE_OPERAND (op, 0) | |
6875 | = maybe_cast_middle_bitint (&gsi, | |
6876 | TREE_OPERAND (op, 0), | |
6877 | type); | |
6878 | TREE_OPERAND (op, 1) | |
6879 | = maybe_cast_middle_bitint (&gsi, | |
6880 | TREE_OPERAND (op, 1), | |
6881 | type); | |
6882 | } | |
6883 | else if (TREE_CODE (op) == CASE_LABEL_EXPR) | |
6884 | { | |
6885 | CASE_LOW (op) | |
6886 | = maybe_cast_middle_bitint (&gsi, CASE_LOW (op), | |
6887 | type); | |
6888 | CASE_HIGH (op) | |
6889 | = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op), | |
6890 | type); | |
6891 | } | |
6892 | } | |
1a84af19 JJ |
6893 | if (tree lhs = gimple_get_lhs (stmt)) |
6894 | if (TREE_CODE (TREE_TYPE (lhs)) == BITINT_TYPE | |
6895 | && (bitint_precision_kind (TREE_TYPE (lhs)) | |
6896 | == bitint_prec_middle)) | |
6897 | { | |
6898 | int prec = TYPE_PRECISION (TREE_TYPE (lhs)); | |
6899 | int uns = TYPE_UNSIGNED (TREE_TYPE (lhs)); | |
6900 | type = build_nonstandard_integer_type (prec, uns); | |
6901 | tree lhs2 = make_ssa_name (type); | |
6902 | gimple_set_lhs (stmt, lhs2); | |
6903 | gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2); | |
6904 | if (stmt_ends_bb_p (stmt)) | |
6905 | { | |
6906 | edge e = find_fallthru_edge (gsi_bb (gsi)->succs); | |
6907 | gsi_insert_on_edge_immediate (e, g); | |
6908 | } | |
6909 | else | |
6910 | gsi_insert_after (&gsi, g, GSI_SAME_STMT); | |
6911 | } | |
a9d6c7fb JJ |
6912 | update_stmt (stmt); |
6913 | continue; | |
6914 | } | |
6915 | ||
6916 | if (tree lhs = gimple_get_lhs (stmt)) | |
6917 | if (TREE_CODE (lhs) == SSA_NAME) | |
6918 | { | |
6919 | tree type = TREE_TYPE (lhs); | |
6920 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
6921 | type = TREE_TYPE (type); | |
6922 | if (TREE_CODE (type) == BITINT_TYPE | |
6923 | && bitint_precision_kind (type) >= bitint_prec_large | |
6924 | && (large_huge.m_names == NULL | |
6925 | || !bitmap_bit_p (large_huge.m_names, | |
6926 | SSA_NAME_VERSION (lhs)))) | |
6927 | continue; | |
6928 | } | |
6929 | ||
6930 | large_huge.lower_stmt (stmt); | |
6931 | } | |
6932 | ||
6933 | tree atype = NULL_TREE; | |
6934 | for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi); | |
6935 | gsi_next (&gsi)) | |
6936 | { | |
6937 | gphi *phi = gsi.phi (); | |
6938 | tree lhs = gimple_phi_result (phi); | |
6939 | if (TREE_CODE (TREE_TYPE (lhs)) != BITINT_TYPE | |
6940 | || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large) | |
6941 | continue; | |
6942 | int p1 = var_to_partition (large_huge.m_map, lhs); | |
6943 | gcc_assert (large_huge.m_vars[p1] != NULL_TREE); | |
6944 | tree v1 = large_huge.m_vars[p1]; | |
6945 | for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i) | |
6946 | { | |
6947 | tree arg = gimple_phi_arg_def (phi, i); | |
6948 | edge e = gimple_phi_arg_edge (phi, i); | |
6949 | gimple *g; | |
6950 | switch (TREE_CODE (arg)) | |
6951 | { | |
6952 | case INTEGER_CST: | |
6953 | if (integer_zerop (arg) && VAR_P (v1)) | |
6954 | { | |
6955 | tree zero = build_zero_cst (TREE_TYPE (v1)); | |
6956 | g = gimple_build_assign (v1, zero); | |
6957 | gsi_insert_on_edge (e, g); | |
6958 | edge_insertions = true; | |
6959 | break; | |
6960 | } | |
6961 | int ext; | |
6962 | unsigned int min_prec, prec, rem; | |
6963 | tree c; | |
6964 | prec = TYPE_PRECISION (TREE_TYPE (arg)); | |
6965 | rem = prec % (2 * limb_prec); | |
6966 | min_prec = bitint_min_cst_precision (arg, ext); | |
6967 | if (min_prec > prec - rem - 2 * limb_prec | |
6968 | && min_prec > (unsigned) limb_prec) | |
6969 | /* Constant which has enough significant bits that it | |
6970 | isn't worth trying to save .rodata space by extending | |
6971 | from smaller number. */ | |
6972 | min_prec = prec; | |
6973 | else | |
6974 | min_prec = CEIL (min_prec, limb_prec) * limb_prec; | |
6975 | if (min_prec == 0) | |
6976 | c = NULL_TREE; | |
6977 | else if (min_prec == prec) | |
6978 | c = tree_output_constant_def (arg); | |
6979 | else if (min_prec == (unsigned) limb_prec) | |
6980 | c = fold_convert (large_huge.m_limb_type, arg); | |
6981 | else | |
6982 | { | |
6983 | tree ctype = build_bitint_type (min_prec, 1); | |
6984 | c = tree_output_constant_def (fold_convert (ctype, arg)); | |
6985 | } | |
6986 | if (c) | |
6987 | { | |
6988 | if (VAR_P (v1) && min_prec == prec) | |
6989 | { | |
6990 | tree v2 = build1 (VIEW_CONVERT_EXPR, | |
6991 | TREE_TYPE (v1), c); | |
6992 | g = gimple_build_assign (v1, v2); | |
6993 | gsi_insert_on_edge (e, g); | |
6994 | edge_insertions = true; | |
6995 | break; | |
6996 | } | |
6997 | if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE) | |
6998 | g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR, | |
6999 | TREE_TYPE (c), v1), | |
7000 | c); | |
7001 | else | |
7002 | { | |
7003 | unsigned HOST_WIDE_INT nelts | |
7004 | = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c))) | |
7005 | / limb_prec; | |
7006 | tree vtype | |
7007 | = build_array_type_nelts (large_huge.m_limb_type, | |
7008 | nelts); | |
7009 | g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR, | |
7010 | vtype, v1), | |
7011 | build1 (VIEW_CONVERT_EXPR, | |
7012 | vtype, c)); | |
7013 | } | |
7014 | gsi_insert_on_edge (e, g); | |
7015 | } | |
7016 | if (ext == 0) | |
7017 | { | |
7018 | unsigned HOST_WIDE_INT nelts | |
7019 | = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1))) | |
7020 | - min_prec) / limb_prec; | |
7021 | tree vtype | |
7022 | = build_array_type_nelts (large_huge.m_limb_type, | |
7023 | nelts); | |
7024 | tree ptype = build_pointer_type (TREE_TYPE (v1)); | |
efef8d7f JJ |
7025 | tree off; |
7026 | if (c) | |
7027 | off = fold_convert (ptype, | |
7028 | TYPE_SIZE_UNIT (TREE_TYPE (c))); | |
7029 | else | |
7030 | off = build_zero_cst (ptype); | |
a9d6c7fb JJ |
7031 | tree vd = build2 (MEM_REF, vtype, |
7032 | build_fold_addr_expr (v1), off); | |
7033 | g = gimple_build_assign (vd, build_zero_cst (vtype)); | |
7034 | } | |
7035 | else | |
7036 | { | |
7037 | tree vd = v1; | |
7038 | if (c) | |
7039 | { | |
7040 | tree ptype = build_pointer_type (TREE_TYPE (v1)); | |
7041 | tree off | |
7042 | = fold_convert (ptype, | |
7043 | TYPE_SIZE_UNIT (TREE_TYPE (c))); | |
7044 | vd = build2 (MEM_REF, large_huge.m_limb_type, | |
7045 | build_fold_addr_expr (v1), off); | |
7046 | } | |
7047 | vd = build_fold_addr_expr (vd); | |
7048 | unsigned HOST_WIDE_INT nbytes | |
7049 | = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1))); | |
7050 | if (c) | |
7051 | nbytes | |
7052 | -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c))); | |
7053 | tree fn = builtin_decl_implicit (BUILT_IN_MEMSET); | |
7054 | g = gimple_build_call (fn, 3, vd, | |
7055 | integer_minus_one_node, | |
7056 | build_int_cst (sizetype, | |
7057 | nbytes)); | |
7058 | } | |
7059 | gsi_insert_on_edge (e, g); | |
7060 | edge_insertions = true; | |
7061 | break; | |
7062 | default: | |
7063 | gcc_unreachable (); | |
7064 | case SSA_NAME: | |
7065 | if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP) | |
7066 | { | |
7067 | if (large_huge.m_names == NULL | |
7068 | || !bitmap_bit_p (large_huge.m_names, | |
7069 | SSA_NAME_VERSION (arg))) | |
7070 | continue; | |
7071 | } | |
7072 | int p2 = var_to_partition (large_huge.m_map, arg); | |
7073 | if (p1 == p2) | |
7074 | continue; | |
7075 | gcc_assert (large_huge.m_vars[p2] != NULL_TREE); | |
7076 | tree v2 = large_huge.m_vars[p2]; | |
7077 | if (VAR_P (v1) && VAR_P (v2)) | |
7078 | g = gimple_build_assign (v1, v2); | |
7079 | else if (VAR_P (v1)) | |
7080 | g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR, | |
7081 | TREE_TYPE (v1), v2)); | |
7082 | else if (VAR_P (v2)) | |
7083 | g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR, | |
7084 | TREE_TYPE (v2), v1), v2); | |
7085 | else | |
7086 | { | |
7087 | if (atype == NULL_TREE | |
7088 | || !tree_int_cst_equal (TYPE_SIZE (atype), | |
7089 | TYPE_SIZE (TREE_TYPE (lhs)))) | |
7090 | { | |
7091 | unsigned HOST_WIDE_INT nelts | |
7092 | = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs))) | |
7093 | / limb_prec; | |
7094 | atype | |
7095 | = build_array_type_nelts (large_huge.m_limb_type, | |
7096 | nelts); | |
7097 | } | |
7098 | g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR, | |
7099 | atype, v1), | |
7100 | build1 (VIEW_CONVERT_EXPR, | |
7101 | atype, v2)); | |
7102 | } | |
7103 | gsi_insert_on_edge (e, g); | |
7104 | edge_insertions = true; | |
7105 | break; | |
7106 | } | |
7107 | } | |
7108 | } | |
7109 | } | |
7110 | ||
7111 | if (large_huge.m_names || has_large_huge) | |
7112 | { | |
7113 | gimple *nop = NULL; | |
7114 | for (i = 0; i < num_ssa_names; ++i) | |
7115 | { | |
7116 | tree s = ssa_name (i); | |
7117 | if (s == NULL_TREE) | |
7118 | continue; | |
7119 | tree type = TREE_TYPE (s); | |
7120 | if (TREE_CODE (type) == COMPLEX_TYPE) | |
7121 | type = TREE_TYPE (type); | |
7122 | if (TREE_CODE (type) == BITINT_TYPE | |
7123 | && bitint_precision_kind (type) >= bitint_prec_large) | |
7124 | { | |
7125 | if (large_huge.m_preserved | |
7126 | && bitmap_bit_p (large_huge.m_preserved, | |
7127 | SSA_NAME_VERSION (s))) | |
7128 | continue; | |
7129 | gimple *g = SSA_NAME_DEF_STMT (s); | |
7130 | if (gimple_code (g) == GIMPLE_NOP) | |
7131 | { | |
7132 | if (SSA_NAME_VAR (s)) | |
7133 | set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE); | |
7134 | release_ssa_name (s); | |
7135 | continue; | |
7136 | } | |
f5198f02 JJ |
7137 | if (gimple_bb (g) == NULL) |
7138 | { | |
7139 | release_ssa_name (s); | |
7140 | continue; | |
7141 | } | |
a9d6c7fb JJ |
7142 | if (gimple_code (g) != GIMPLE_ASM) |
7143 | { | |
7144 | gimple_stmt_iterator gsi = gsi_for_stmt (g); | |
7145 | bool save_vta = flag_var_tracking_assignments; | |
7146 | flag_var_tracking_assignments = false; | |
7147 | gsi_remove (&gsi, true); | |
7148 | flag_var_tracking_assignments = save_vta; | |
7149 | } | |
7150 | if (nop == NULL) | |
7151 | nop = gimple_build_nop (); | |
7152 | SSA_NAME_DEF_STMT (s) = nop; | |
7153 | release_ssa_name (s); | |
7154 | } | |
7155 | } | |
7156 | if (optimize) | |
7157 | disable_ranger (cfun); | |
7158 | } | |
7159 | ||
7160 | if (edge_insertions) | |
7161 | gsi_commit_edge_inserts (); | |
7162 | ||
90b98723 JJ |
7163 | /* Fix up arguments of ECF_RETURNS_TWICE calls. Those were temporarily |
7164 | inserted before the call, but that is invalid IL, so move them to the | |
7165 | right place and add corresponding PHIs. */ | |
7166 | if (!large_huge.m_returns_twice_calls.is_empty ()) | |
7167 | { | |
7168 | auto_vec<gimple *, 16> arg_stmts; | |
7169 | while (!large_huge.m_returns_twice_calls.is_empty ()) | |
7170 | { | |
7171 | gimple *stmt = large_huge.m_returns_twice_calls.pop (); | |
7172 | gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt)); | |
7173 | while (gsi_stmt (gsi) != stmt) | |
7174 | { | |
7dd1f9d2 JJ |
7175 | if (is_gimple_debug (gsi_stmt (gsi))) |
7176 | gsi_next (&gsi); | |
7177 | else | |
7178 | { | |
7179 | arg_stmts.safe_push (gsi_stmt (gsi)); | |
7180 | gsi_remove (&gsi, false); | |
7181 | } | |
90b98723 JJ |
7182 | } |
7183 | gimple *g; | |
7184 | basic_block bb = NULL; | |
7185 | edge e = NULL, ead = NULL; | |
7186 | FOR_EACH_VEC_ELT (arg_stmts, i, g) | |
7187 | { | |
7188 | gsi_safe_insert_before (&gsi, g); | |
7189 | if (i == 0) | |
7190 | { | |
7191 | bb = gimple_bb (stmt); | |
7192 | gcc_checking_assert (EDGE_COUNT (bb->preds) == 2); | |
7193 | e = EDGE_PRED (bb, 0); | |
7194 | ead = EDGE_PRED (bb, 1); | |
7195 | if ((ead->flags & EDGE_ABNORMAL) == 0) | |
7196 | std::swap (e, ead); | |
7197 | gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0 | |
7198 | && (ead->flags & EDGE_ABNORMAL)); | |
7199 | } | |
7200 | tree lhs = gimple_assign_lhs (g); | |
7201 | tree arg = lhs; | |
7202 | gphi *phi = create_phi_node (copy_ssa_name (arg), bb); | |
7203 | add_phi_arg (phi, arg, e, UNKNOWN_LOCATION); | |
7204 | tree var = create_tmp_reg (TREE_TYPE (arg)); | |
7205 | suppress_warning (var, OPT_Wuninitialized); | |
7206 | arg = get_or_create_ssa_default_def (cfun, var); | |
7207 | SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1; | |
7208 | add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION); | |
7209 | arg = gimple_phi_result (phi); | |
7210 | SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1; | |
7211 | imm_use_iterator iter; | |
7212 | gimple *use_stmt; | |
7213 | FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs) | |
7214 | { | |
7215 | if (use_stmt == phi) | |
7216 | continue; | |
7217 | gcc_checking_assert (use_stmt == stmt); | |
7218 | use_operand_p use_p; | |
7219 | FOR_EACH_IMM_USE_ON_STMT (use_p, iter) | |
7220 | SET_USE (use_p, arg); | |
7221 | } | |
7222 | } | |
7223 | update_stmt (stmt); | |
7224 | arg_stmts.truncate (0); | |
7225 | } | |
7226 | } | |
7227 | ||
a9d6c7fb JJ |
7228 | return ret; |
7229 | } | |
7230 | ||
7231 | namespace { | |
7232 | ||
7233 | const pass_data pass_data_lower_bitint = | |
7234 | { | |
7235 | GIMPLE_PASS, /* type */ | |
7236 | "bitintlower", /* name */ | |
7237 | OPTGROUP_NONE, /* optinfo_flags */ | |
7238 | TV_NONE, /* tv_id */ | |
7239 | PROP_ssa, /* properties_required */ | |
7240 | PROP_gimple_lbitint, /* properties_provided */ | |
7241 | 0, /* properties_destroyed */ | |
7242 | 0, /* todo_flags_start */ | |
7243 | 0, /* todo_flags_finish */ | |
7244 | }; | |
7245 | ||
7246 | class pass_lower_bitint : public gimple_opt_pass | |
7247 | { | |
7248 | public: | |
7249 | pass_lower_bitint (gcc::context *ctxt) | |
7250 | : gimple_opt_pass (pass_data_lower_bitint, ctxt) | |
7251 | {} | |
7252 | ||
7253 | /* opt_pass methods: */ | |
7254 | opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); } | |
7255 | unsigned int execute (function *) final override | |
7256 | { | |
7257 | return gimple_lower_bitint (); | |
7258 | } | |
7259 | ||
7260 | }; // class pass_lower_bitint | |
7261 | ||
7262 | } // anon namespace | |
7263 | ||
7264 | gimple_opt_pass * | |
7265 | make_pass_lower_bitint (gcc::context *ctxt) | |
7266 | { | |
7267 | return new pass_lower_bitint (ctxt); | |
7268 | } | |
7269 | ||
7270 | \f | |
7271 | namespace { | |
7272 | ||
7273 | const pass_data pass_data_lower_bitint_O0 = | |
7274 | { | |
7275 | GIMPLE_PASS, /* type */ | |
7276 | "bitintlower0", /* name */ | |
7277 | OPTGROUP_NONE, /* optinfo_flags */ | |
7278 | TV_NONE, /* tv_id */ | |
7279 | PROP_cfg, /* properties_required */ | |
7280 | PROP_gimple_lbitint, /* properties_provided */ | |
7281 | 0, /* properties_destroyed */ | |
7282 | 0, /* todo_flags_start */ | |
7283 | 0, /* todo_flags_finish */ | |
7284 | }; | |
7285 | ||
7286 | class pass_lower_bitint_O0 : public gimple_opt_pass | |
7287 | { | |
7288 | public: | |
7289 | pass_lower_bitint_O0 (gcc::context *ctxt) | |
7290 | : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt) | |
7291 | {} | |
7292 | ||
7293 | /* opt_pass methods: */ | |
7294 | bool gate (function *fun) final override | |
7295 | { | |
7296 | /* With errors, normal optimization passes are not run. If we don't | |
7297 | lower bitint operations at all, rtl expansion will abort. */ | |
7298 | return !(fun->curr_properties & PROP_gimple_lbitint); | |
7299 | } | |
7300 | ||
7301 | unsigned int execute (function *) final override | |
7302 | { | |
7303 | return gimple_lower_bitint (); | |
7304 | } | |
7305 | ||
7306 | }; // class pass_lower_bitint_O0 | |
7307 | ||
7308 | } // anon namespace | |
7309 | ||
7310 | gimple_opt_pass * | |
7311 | make_pass_lower_bitint_O0 (gcc::context *ctxt) | |
7312 | { | |
7313 | return new pass_lower_bitint_O0 (ctxt); | |
7314 | } |