1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "diagnostic.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
34 #include "langhooks.h"
36 #include "gimple-iterator.h"
37 #include "gimplify-me.h"
40 #include "tree-vector-builder.h"
41 #include "vec-perm-indices.h"
42 #include "insn-config.h"
43 #include "tree-ssa-dce.h"
44 #include "gimple-fold.h"
45 #include "gimple-match.h"
46 #include "recog.h" /* FIXME: for insn_data */
47 #include "optabs-libfuncs.h"
50 /* Build a ternary operation and gimplify it. Emit code before GSI.
51 Return the gimple_val holding the result. */
54 gimplify_build3 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
55 tree type
, tree a
, tree b
, tree c
)
57 location_t loc
= gimple_location (gsi_stmt (*gsi
));
58 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
, c
);
61 /* Build a binary operation and gimplify it. Emit code before GSI.
62 Return the gimple_val holding the result. */
65 gimplify_build2 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
66 tree type
, tree a
, tree b
)
68 location_t loc
= gimple_location (gsi_stmt (*gsi
));
69 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
);
72 /* Build a unary operation and gimplify it. Emit code before GSI.
73 Return the gimple_val holding the result. */
76 gimplify_build1 (gimple_stmt_iterator
*gsi
, enum tree_code code
, tree type
,
79 location_t loc
= gimple_location (gsi_stmt (*gsi
));
80 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
);
84 static void expand_vector_operations_1 (gimple_stmt_iterator
*, bitmap
);
86 /* Return the number of elements in a vector type TYPE that we have
87 already decided needs to be expanded piecewise. We don't support
88 this kind of expansion for variable-length vectors, since we should
89 always check for target support before introducing uses of those. */
91 nunits_for_known_piecewise_op (const_tree type
)
93 return TYPE_VECTOR_SUBPARTS (type
).to_constant ();
96 /* Return true if TYPE1 has more elements than TYPE2, where either
97 type may be a vector or a scalar. */
100 subparts_gt (tree type1
, tree type2
)
102 poly_uint64 n1
= VECTOR_TYPE_P (type1
) ? TYPE_VECTOR_SUBPARTS (type1
) : 1;
103 poly_uint64 n2
= VECTOR_TYPE_P (type2
) ? TYPE_VECTOR_SUBPARTS (type2
) : 1;
104 return known_gt (n1
, n2
);
107 static GTY(()) tree vector_inner_type
;
108 static GTY(()) tree vector_last_type
;
109 static GTY(()) int vector_last_nunits
;
111 /* Return a suitable vector types made of SUBPARTS units each of mode
112 "word_mode" (the global variable). */
114 build_word_mode_vector_type (int nunits
)
116 if (!vector_inner_type
)
117 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
118 else if (vector_last_nunits
== nunits
)
120 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
121 return vector_last_type
;
124 vector_last_nunits
= nunits
;
125 vector_last_type
= build_vector_type (vector_inner_type
, nunits
);
126 return vector_last_type
;
129 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
130 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
133 /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
137 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
138 tree t
, tree bitsize
, tree bitpos
)
140 /* We're using the resimplify API and maybe_push_res_to_seq to
141 simplify the BIT_FIELD_REF but restrict the simplification to
142 a single stmt while at the same time following SSA edges for
143 simplification with already emitted CTORs. */
145 opr
.set_op (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
146 opr
.resimplify (NULL
, follow_all_ssa_edges
);
147 gimple_seq stmts
= NULL
;
148 tree res
= maybe_push_res_to_seq (&opr
, &stmts
);
151 /* This can happen if SSA_NAME_OCCURS_IN_ABNORMAL_PHI are
152 used. Build BIT_FIELD_REF manually otherwise. */
153 t
= build3 (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
154 res
= make_ssa_name (type
);
155 gimple
*g
= gimple_build_assign (res
, t
);
156 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
159 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
164 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
165 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
166 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
168 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
169 return gimplify_build1 (gsi
, code
, inner_type
, a
);
173 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
174 tree bitpos
, tree bitsize
, enum tree_code code
,
175 tree type ATTRIBUTE_UNUSED
)
177 if (VECTOR_TYPE_P (TREE_TYPE (a
)))
178 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
179 if (VECTOR_TYPE_P (TREE_TYPE (b
)))
180 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
181 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
184 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
186 INNER_TYPE is the type of A and B elements
188 returned expression is of signed integer type with the
189 size equal to the size of INNER_TYPE. */
191 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
192 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
194 tree stype
= TREE_TYPE (type
);
195 tree cst_false
= build_zero_cst (stype
);
196 tree cst_true
= build_all_ones_cst (stype
);
199 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
200 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
202 cmp
= build2 (code
, boolean_type_node
, a
, b
);
203 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
206 /* Expand vector addition to scalars. This does bit twiddling
207 in order to increase parallelism:
209 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
212 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
213 (a ^ ~b) & 0x80808080
215 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
217 This optimization should be done only if 4 vector items or more
220 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
221 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
222 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
224 unsigned int width
= vector_element_bits (TREE_TYPE (a
));
225 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
226 unsigned HOST_WIDE_INT max
;
227 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
229 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
230 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
231 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
233 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
234 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
236 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
237 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
238 if (code
== PLUS_EXPR
)
239 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
242 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
243 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
246 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
247 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
248 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
252 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
253 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
254 tree bitsize ATTRIBUTE_UNUSED
,
255 enum tree_code code ATTRIBUTE_UNUSED
,
256 tree type ATTRIBUTE_UNUSED
)
258 unsigned int width
= vector_element_bits (TREE_TYPE (b
));
259 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
261 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
263 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
264 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
265 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
267 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
269 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
270 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
271 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
272 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
273 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
276 /* Expand a vector operation to scalars, by using many operations
277 whose type is the vector type's inner type. */
279 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
280 tree type
, tree inner_type
,
281 tree a
, tree b
, enum tree_code code
,
282 bool parallel_p
, tree ret_type
= NULL_TREE
)
284 vec
<constructor_elt
, va_gc
> *v
;
285 tree part_width
= TYPE_SIZE (inner_type
);
286 tree index
= bitsize_int (0);
287 int nunits
= nunits_for_known_piecewise_op (type
);
288 int delta
= tree_to_uhwi (part_width
) / vector_element_bits (type
);
290 location_t loc
= gimple_location (gsi_stmt (*gsi
));
293 || warning_suppressed_p (gsi_stmt (*gsi
),
294 OPT_Wvector_operation_performance
))
295 /* Do not diagnose decomposing single element vectors or when
296 decomposing vectorizer produced operations. */
298 else if (ret_type
|| !parallel_p
)
299 warning_at (loc
, OPT_Wvector_operation_performance
,
300 "vector operation will be expanded piecewise");
302 warning_at (loc
, OPT_Wvector_operation_performance
,
303 "vector operation will be expanded in parallel");
307 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
308 bool constant_p
= true;
309 for (i
= 0; i
< nunits
;
310 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
312 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
,
314 if (!CONSTANT_CLASS_P (result
))
316 constructor_elt ce
= {NULL_TREE
, result
};
321 return build_vector_from_ctor (ret_type
, v
);
323 return build_constructor (ret_type
, v
);
326 /* Expand a vector operation to scalars with the freedom to use
327 a scalar integer type, or to use a different size for the items
328 in the vector type. */
330 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
331 tree a
, tree b
, enum tree_code code
)
333 tree result
, compute_type
;
334 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
335 location_t loc
= gimple_location (gsi_stmt (*gsi
));
337 /* We have three strategies. If the type is already correct, just do
338 the operation an element at a time. Else, if the vector is wider than
339 one word, do it a word at a time; finally, if the vector is smaller
340 than one word, do it as a scalar. */
341 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
342 return expand_vector_piecewise (gsi
, f
,
343 type
, TREE_TYPE (type
),
345 else if (n_words
> 1)
347 tree word_type
= build_word_mode_vector_type (n_words
);
348 result
= expand_vector_piecewise (gsi
, f
,
349 word_type
, TREE_TYPE (word_type
),
351 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
356 /* Use a single scalar operation with a mode no wider than word_mode. */
357 if (!warning_suppressed_p (gsi_stmt (*gsi
),
358 OPT_Wvector_operation_performance
))
359 warning_at (loc
, OPT_Wvector_operation_performance
,
360 "vector operation will be expanded with a "
361 "single scalar operation");
363 = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), 0).require ();
364 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
365 result
= f (gsi
, compute_type
, a
, b
, bitsize_zero_node
,
366 TYPE_SIZE (compute_type
), code
, type
);
372 /* Expand a vector operation to scalars; for integer types we can use
373 special bit twiddling tricks to do the sums a word at a time, using
374 function F_PARALLEL instead of F. These tricks are done only if
375 they can process at least four items, that is, only if the vector
376 holds at least four items and if a word can hold four items. */
378 expand_vector_addition (gimple_stmt_iterator
*gsi
,
379 elem_op_func f
, elem_op_func f_parallel
,
380 tree type
, tree a
, tree b
, enum tree_code code
)
382 int parts_per_word
= BITS_PER_WORD
/ vector_element_bits (type
);
384 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
385 && parts_per_word
>= 4
386 && nunits_for_known_piecewise_op (type
) >= 4)
387 return expand_vector_parallel (gsi
, f_parallel
,
390 return expand_vector_piecewise (gsi
, f
,
391 type
, TREE_TYPE (type
),
396 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
);
398 /* Try to expand vector comparison expression OP0 CODE OP1 by
399 querying optab if the following expression:
400 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
403 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
404 tree op1
, enum tree_code code
,
405 bitmap dce_ssa_names
)
407 tree lhs
= gimple_assign_lhs (gsi_stmt (*gsi
));
409 imm_use_iterator iterator
;
410 bool vec_cond_expr_only
= true;
412 /* As seen in PR95830, we should not expand comparisons that are only
413 feeding a VEC_COND_EXPR statement. */
414 auto_vec
<gimple
*> uses
;
415 FOR_EACH_IMM_USE_FAST (use_p
, iterator
, lhs
)
417 gimple
*use
= USE_STMT (use_p
);
418 if (is_gimple_debug (use
))
420 if (is_gimple_assign (use
)
421 && gimple_assign_rhs_code (use
) == VEC_COND_EXPR
422 && gimple_assign_rhs1 (use
) == lhs
423 && gimple_assign_rhs2 (use
) != lhs
424 && gimple_assign_rhs3 (use
) != lhs
)
425 uses
.safe_push (use
);
427 vec_cond_expr_only
= false;
430 if (vec_cond_expr_only
)
431 for (gimple
*use
: uses
)
433 gimple_stmt_iterator it
= gsi_for_stmt (use
);
434 if (!expand_vector_condition (&it
, dce_ssa_names
))
436 vec_cond_expr_only
= false;
441 if (!uses
.is_empty () && vec_cond_expr_only
)
445 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
, code
))
447 if (VECTOR_BOOLEAN_TYPE_P (type
)
448 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
449 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
450 TYPE_VECTOR_SUBPARTS (type
)
451 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
452 (TREE_TYPE (type
)))))
454 tree inner_type
= TREE_TYPE (TREE_TYPE (op0
));
455 tree part_width
= vector_element_bits_tree (TREE_TYPE (op0
));
456 tree index
= bitsize_int (0);
457 int nunits
= nunits_for_known_piecewise_op (TREE_TYPE (op0
));
458 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (type
));
459 tree ret_type
= build_nonstandard_integer_type (prec
, 1);
460 tree ret_inner_type
= boolean_type_node
;
462 location_t loc
= gimple_location (gsi_stmt (*gsi
));
463 t
= build_zero_cst (ret_type
);
465 if (TYPE_PRECISION (ret_inner_type
) != 1)
466 ret_inner_type
= build_nonstandard_integer_type (1, 1);
467 if (!warning_suppressed_p (gsi_stmt (*gsi
),
468 OPT_Wvector_operation_performance
))
469 warning_at (loc
, OPT_Wvector_operation_performance
,
470 "vector operation will be expanded piecewise");
471 for (i
= 0; i
< nunits
;
472 i
++, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
474 tree a
= tree_vec_extract (gsi
, inner_type
, op0
, part_width
,
476 tree b
= tree_vec_extract (gsi
, inner_type
, op1
, part_width
,
478 tree result
= gimplify_build2 (gsi
, code
, ret_inner_type
, a
, b
);
479 t
= gimplify_build3 (gsi
, BIT_INSERT_EXPR
, ret_type
, t
, result
,
482 t
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
485 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
486 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
,
495 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
496 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
497 the result if successful, otherwise return NULL_TREE. */
499 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
502 unsigned int i
, nunits
= nunits_for_known_piecewise_op (type
);
503 bool scalar_shift
= true;
505 for (i
= 1; i
< nunits
; i
++)
507 if (shiftcnts
[i
] != shiftcnts
[0])
508 scalar_shift
= false;
511 if (scalar_shift
&& shiftcnts
[0] == 0)
516 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
517 if (op
!= unknown_optab
518 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
519 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
520 build_int_cst (NULL_TREE
, shiftcnts
[0]));
523 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
524 if (op
!= unknown_optab
525 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
527 tree_vector_builder
vec (type
, nunits
, 1);
528 for (i
= 0; i
< nunits
; i
++)
529 vec
.quick_push (build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]));
530 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
, vec
.build ());
536 /* Try to expand integer vector division by constant using
537 widening multiply, shifts and additions. */
539 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
540 tree op1
, enum tree_code code
)
542 bool use_pow2
= true;
543 bool has_vector_shift
= true;
544 bool use_abs_op1
= false;
545 int mode
= -1, this_mode
;
546 int pre_shift
= -1, post_shift
;
547 unsigned int nunits
= nunits_for_known_piecewise_op (type
);
548 int *shifts
= XALLOCAVEC (int, nunits
* 4);
549 int *pre_shifts
= shifts
+ nunits
;
550 int *post_shifts
= pre_shifts
+ nunits
;
551 int *shift_temps
= post_shifts
+ nunits
;
552 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
553 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
555 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
556 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
557 tree cur_op
, mulcst
, tem
;
560 if (prec
> HOST_BITS_PER_WIDE_INT
)
563 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
564 if (op
== unknown_optab
565 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
566 has_vector_shift
= false;
568 /* Analysis phase. Determine if all op1 elements are either power
569 of two and it is possible to expand it using shifts (or for remainder
570 using masking). Additionally compute the multiplicative constants
571 and pre and post shifts if the division is to be expanded using
572 widening or high part multiplication plus shifts. */
573 for (i
= 0; i
< nunits
; i
++)
575 tree cst
= VECTOR_CST_ELT (op1
, i
);
576 unsigned HOST_WIDE_INT ml
;
578 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
584 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
588 shifts
[i
] = tree_log2 (cst
);
589 if (shifts
[i
] != shifts
[0]
590 && code
== TRUNC_DIV_EXPR
591 && !has_vector_shift
)
596 if (sign_p
== UNSIGNED
)
598 unsigned HOST_WIDE_INT mh
;
599 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
601 if (d
>= (HOST_WIDE_INT_1U
<< (prec
- 1)))
602 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
611 /* Find a suitable multiplier and right shift count instead of
612 directly dividing by D. */
613 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
);
615 /* If the suggested multiplier is more than PREC bits, we can
616 do better for even divisors, using an initial right shift. */
617 if ((mh
!= 0 && (d
& 1) == 0)
618 || (!has_vector_shift
&& pre_shift
!= -1))
620 if (has_vector_shift
)
621 pre_shift
= ctz_or_zero (d
);
622 else if (pre_shift
== -1)
625 for (j
= 0; j
< nunits
; j
++)
627 tree cst2
= VECTOR_CST_ELT (op1
, j
);
628 unsigned HOST_WIDE_INT d2
;
631 if (!tree_fits_uhwi_p (cst2
))
633 d2
= tree_to_uhwi (cst2
) & mask
;
636 this_pre_shift
= floor_log2 (d2
& -d2
);
637 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
638 pre_shift
= this_pre_shift
;
640 if (i
!= 0 && pre_shift
!= 0)
650 if ((d
>> pre_shift
) <= 1)
655 mh
= choose_multiplier (d
>> pre_shift
, prec
,
659 pre_shifts
[i
] = pre_shift
;
669 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
670 unsigned HOST_WIDE_INT abs_d
;
675 /* Since d might be INT_MIN, we have to cast to
676 unsigned HOST_WIDE_INT before negating to avoid
677 undefined signed overflow. */
679 ? (unsigned HOST_WIDE_INT
) d
680 : - (unsigned HOST_WIDE_INT
) d
);
682 /* n rem d = n rem -d */
683 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
688 if (abs_d
== HOST_WIDE_INT_1U
<< (prec
- 1))
690 /* This case is not handled correctly below. */
700 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
702 if (ml
>= HOST_WIDE_INT_1U
<< (prec
- 1))
704 this_mode
= 4 + (d
< 0);
705 ml
|= HOST_WIDE_INT_M1U
<< (prec
- 1);
708 this_mode
= 2 + (d
< 0);
711 post_shifts
[i
] = post_shift
;
712 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
713 || post_shift
>= prec
714 || pre_shifts
[i
] >= prec
)
719 else if (mode
!= this_mode
)
725 tree addend
= NULL_TREE
;
726 if (sign_p
== SIGNED
)
730 /* Both division and remainder sequences need
731 op0 < 0 ? mask : 0 computed. It can be either computed as
732 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
733 if none of the shifts is 0, or as the conditional. */
734 for (i
= 0; i
< nunits
; i
++)
738 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
740 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
742 for (i
= 0; i
< nunits
; i
++)
743 shift_temps
[i
] = prec
- 1;
744 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
745 if (cur_op
!= NULL_TREE
)
747 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
749 for (i
= 0; i
< nunits
; i
++)
750 shift_temps
[i
] = prec
- shifts
[i
];
751 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
752 if (cur_op
!= NULL_TREE
)
753 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
757 if (addend
== NULL_TREE
758 && expand_vec_cond_expr_p (type
, type
, LT_EXPR
))
760 tree zero
, cst
, mask_type
, mask
;
763 mask_type
= truth_type_for (type
);
764 zero
= build_zero_cst (type
);
765 mask
= make_ssa_name (mask_type
);
766 cond
= gimple_build_assign (mask
, LT_EXPR
, op0
, zero
);
767 gsi_insert_before (gsi
, cond
, GSI_SAME_STMT
);
768 tree_vector_builder
vec (type
, nunits
, 1);
769 for (i
= 0; i
< nunits
; i
++)
770 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
774 addend
= make_ssa_name (type
);
776 = gimple_build_assign (addend
, VEC_COND_EXPR
, mask
, cst
, zero
);
777 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
780 if (code
== TRUNC_DIV_EXPR
)
782 if (sign_p
== UNSIGNED
)
784 /* q = op0 >> shift; */
785 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
786 if (cur_op
!= NULL_TREE
)
789 else if (addend
!= NULL_TREE
)
791 /* t1 = op0 + addend;
793 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
794 if (op
!= unknown_optab
795 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
797 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
798 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
799 if (cur_op
!= NULL_TREE
)
807 tree_vector_builder
vec (type
, nunits
, 1);
808 for (i
= 0; i
< nunits
; i
++)
809 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
813 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
814 if (op
!= unknown_optab
815 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
817 if (sign_p
== UNSIGNED
)
818 /* r = op0 & mask; */
819 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
820 else if (addend
!= NULL_TREE
)
822 /* t1 = op0 + addend;
825 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
826 if (op
!= unknown_optab
827 && optab_handler (op
, TYPE_MODE (type
))
830 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
832 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
834 op
= optab_for_tree_code (MINUS_EXPR
, type
,
836 if (op
!= unknown_optab
837 && optab_handler (op
, TYPE_MODE (type
))
839 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
847 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
850 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
858 gcc_assert (sign_p
== UNSIGNED
);
859 /* t1 = oprnd0 >> pre_shift;
861 q = t2 >> post_shift; */
862 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
863 if (cur_op
== NULL_TREE
)
867 gcc_assert (sign_p
== UNSIGNED
);
868 for (i
= 0; i
< nunits
; i
++)
878 gcc_assert (sign_p
== SIGNED
);
879 for (i
= 0; i
< nunits
; i
++)
880 shift_temps
[i
] = prec
- 1;
886 tree_vector_builder
vec (type
, nunits
, 1);
887 for (i
= 0; i
< nunits
; i
++)
888 vec
.quick_push (build_int_cst (TREE_TYPE (type
), mulc
[i
]));
889 mulcst
= vec
.build ();
891 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
896 /* t1 = oprnd0 >> pre_shift;
898 q = t2 >> post_shift; */
899 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
902 /* t1 = oprnd0 h* ml;
906 q = t4 >> (post_shift - 1); */
907 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
908 if (op
== unknown_optab
909 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
911 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
912 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
913 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
914 if (op
== unknown_optab
915 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
917 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
918 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
919 if (cur_op
== NULL_TREE
)
926 /* t1 = oprnd0 h* ml;
927 t2 = t1; [ iff (mode & 2) != 0 ]
928 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
929 t3 = t2 >> post_shift;
930 t4 = oprnd0 >> (prec - 1);
931 q = t3 - t4; [ iff (mode & 1) == 0 ]
932 q = t4 - t3; [ iff (mode & 1) != 0 ] */
935 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
936 if (op
== unknown_optab
937 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
939 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
941 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
942 if (cur_op
== NULL_TREE
)
944 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
945 if (tem
== NULL_TREE
)
947 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
948 if (op
== unknown_optab
949 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
952 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
954 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
960 if (code
== TRUNC_DIV_EXPR
)
963 /* We divided. Now finish by:
966 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
967 if (op
== unknown_optab
968 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
972 tree_vector_builder elts
;
973 if (!elts
.new_unary_operation (type
, op1
, false))
975 unsigned int count
= elts
.encoded_nelts ();
976 for (unsigned int i
= 0; i
< count
; ++i
)
978 tree elem1
= VECTOR_CST_ELT (op1
, i
);
980 tree elt
= const_unop (ABS_EXPR
, TREE_TYPE (elem1
), elem1
);
981 if (elt
== NULL_TREE
)
983 elts
.quick_push (elt
);
987 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
988 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
989 if (op
== unknown_optab
990 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
992 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
995 /* Expand a vector condition to scalars, by using many conditions
996 on the vector's elements. */
999 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
)
1001 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1002 tree type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1003 tree a
= gimple_assign_rhs1 (stmt
);
1005 tree a2
= NULL_TREE
;
1006 bool a_is_comparison
= false;
1007 bool a_is_scalar_bitmask
= false;
1008 tree b
= gimple_assign_rhs2 (stmt
);
1009 tree c
= gimple_assign_rhs3 (stmt
);
1010 vec
<constructor_elt
, va_gc
> *v
;
1012 tree inner_type
= TREE_TYPE (type
);
1013 tree width
= vector_element_bits_tree (type
);
1014 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
1015 tree comp_inner_type
= cond_type
;
1016 tree index
= bitsize_int (0);
1017 tree comp_width
= width
;
1018 tree comp_index
= index
;
1019 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1020 tree_code code
= TREE_CODE (a
);
1021 gassign
*assign
= NULL
;
1023 if (code
== SSA_NAME
)
1025 assign
= dyn_cast
<gassign
*> (SSA_NAME_DEF_STMT (a
));
1027 && TREE_CODE_CLASS (gimple_assign_rhs_code (assign
)) == tcc_comparison
)
1029 a_is_comparison
= true;
1030 a1
= gimple_assign_rhs1 (assign
);
1031 a2
= gimple_assign_rhs2 (assign
);
1032 code
= gimple_assign_rhs_code (assign
);
1033 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
1034 comp_width
= vector_element_bits_tree (TREE_TYPE (a1
));
1038 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
), code
)
1039 || (integer_all_onesp (b
) && integer_zerop (c
)
1040 && expand_vec_cmp_expr_p (type
, TREE_TYPE (a1
), code
)))
1042 gcc_assert (TREE_CODE (a
) == SSA_NAME
|| TREE_CODE (a
) == VECTOR_CST
);
1046 /* If a has vector boolean type and is a comparison, above
1047 expand_vec_cond_expr_p might fail, even if both the comparison and
1048 VEC_COND_EXPR could be supported individually. See PR109176. */
1050 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1051 && expand_vec_cond_expr_p (type
, TREE_TYPE (a
), SSA_NAME
)
1052 && expand_vec_cmp_expr_p (TREE_TYPE (a1
), TREE_TYPE (a
), code
))
1055 /* Handle vector boolean types with bitmasks. If there is a comparison
1056 and we can expand the comparison into the vector boolean bitmask,
1057 or otherwise if it is compatible with type, we can transform
1058 vbfld_1 = x_2 < y_3 ? vbfld_4 : vbfld_5;
1061 tmp_7 = tmp_6 & vbfld_4;
1063 tmp_9 = tmp_8 & vbfld_5;
1064 vbfld_1 = tmp_7 | tmp_9;
1065 Similarly for vbfld_10 instead of x_2 < y_3. */
1066 if (VECTOR_BOOLEAN_TYPE_P (type
)
1067 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
1068 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
1069 TYPE_VECTOR_SUBPARTS (type
)
1070 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (type
))))
1072 ? useless_type_conversion_p (type
, TREE_TYPE (a
))
1073 : expand_vec_cmp_expr_p (TREE_TYPE (a1
), type
, TREE_CODE (a
))))
1075 if (a_is_comparison
)
1076 a
= gimplify_build2 (gsi
, code
, type
, a1
, a2
);
1077 a1
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a
, b
);
1078 a2
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, type
, a
);
1079 a2
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a2
, c
);
1080 a
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, type
, a1
, a2
);
1081 gimple_assign_set_rhs_from_tree (gsi
, a
);
1082 update_stmt (gsi_stmt (*gsi
));
1086 /* TODO: try and find a smaller vector type. */
1088 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1089 warning_at (loc
, OPT_Wvector_operation_performance
,
1090 "vector condition will be expanded piecewise");
1092 if (!a_is_comparison
1093 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1094 && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a
)))
1095 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a
))),
1096 TYPE_VECTOR_SUBPARTS (TREE_TYPE (a
))
1097 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1098 (TREE_TYPE (TREE_TYPE (a
))))))
1100 a_is_scalar_bitmask
= true;
1101 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a
)));
1102 tree atype
= build_nonstandard_integer_type (prec
, 1);
1103 a
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, atype
, a
);
1105 else if (!a_is_comparison
1106 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
)))
1107 comp_width
= vector_element_bits_tree (TREE_TYPE (a
));
1109 int nunits
= nunits_for_known_piecewise_op (type
);
1110 vec_alloc (v
, nunits
);
1111 bool constant_p
= true;
1112 for (int i
= 0; i
< nunits
; i
++)
1115 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
1116 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
1117 if (a_is_comparison
)
1119 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
,
1120 comp_width
, comp_index
);
1121 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
,
1122 comp_width
, comp_index
);
1123 aa
= gimplify_build2 (gsi
, code
, boolean_type_node
, aa1
, aa2
);
1125 else if (a_is_scalar_bitmask
)
1127 wide_int w
= wi::set_bit_in_zero (i
, TYPE_PRECISION (TREE_TYPE (a
)));
1128 result
= gimplify_build2 (gsi
, BIT_AND_EXPR
, TREE_TYPE (a
),
1129 a
, wide_int_to_tree (TREE_TYPE (a
), w
));
1130 aa
= gimplify_build2 (gsi
, NE_EXPR
, boolean_type_node
, result
,
1131 build_zero_cst (TREE_TYPE (a
)));
1135 result
= tree_vec_extract (gsi
, cond_type
, a
, comp_width
, comp_index
);
1136 aa
= gimplify_build2 (gsi
, NE_EXPR
, boolean_type_node
, result
,
1137 build_zero_cst (cond_type
));
1139 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
1140 if (!CONSTANT_CLASS_P (result
))
1142 constructor_elt ce
= {NULL_TREE
, result
};
1144 index
= int_const_binop (PLUS_EXPR
, index
, width
);
1145 if (width
== comp_width
)
1148 comp_index
= int_const_binop (PLUS_EXPR
, comp_index
, comp_width
);
1152 constr
= build_vector_from_ctor (type
, v
);
1154 constr
= build_constructor (type
, v
);
1155 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1156 update_stmt (gsi_stmt (*gsi
));
1158 if (a_is_comparison
)
1159 bitmap_set_bit (dce_ssa_names
,
1160 SSA_NAME_VERSION (gimple_assign_lhs (assign
)));
1166 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
1167 gassign
*assign
, enum tree_code code
,
1168 bitmap dce_ssa_names
)
1170 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1172 /* If the compute mode is not a vector mode (hence we are not decomposing
1173 a BLKmode vector to smaller, hardware-supported vectors), we may want
1174 to expand the operations in parallel. */
1175 if (!VECTOR_MODE_P (compute_mode
))
1180 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1181 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
1182 gimple_assign_rhs1 (assign
),
1183 gimple_assign_rhs2 (assign
), code
);
1187 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1188 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
1189 gimple_assign_rhs1 (assign
),
1196 return expand_vector_parallel (gsi
, do_binop
, type
,
1197 gimple_assign_rhs1 (assign
),
1198 gimple_assign_rhs2 (assign
), code
);
1201 return expand_vector_parallel (gsi
, do_unop
, type
,
1202 gimple_assign_rhs1 (assign
),
1217 case UNORDERED_EXPR
:
1219 tree rhs1
= gimple_assign_rhs1 (assign
);
1220 tree rhs2
= gimple_assign_rhs2 (assign
);
1222 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
,
1226 case TRUNC_DIV_EXPR
:
1227 case TRUNC_MOD_EXPR
:
1229 tree rhs1
= gimple_assign_rhs1 (assign
);
1230 tree rhs2
= gimple_assign_rhs2 (assign
);
1234 || !VECTOR_INTEGER_TYPE_P (type
)
1235 || TREE_CODE (rhs2
) != VECTOR_CST
1236 || !VECTOR_MODE_P (TYPE_MODE (type
)))
1239 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
1240 if (ret
!= NULL_TREE
)
1249 if (TREE_CODE_CLASS (code
) == tcc_unary
)
1250 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
1251 gimple_assign_rhs1 (assign
),
1252 NULL_TREE
, code
, false);
1254 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
1255 gimple_assign_rhs1 (assign
),
1256 gimple_assign_rhs2 (assign
), code
, false);
1260 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1262 _9 = { b_7, b_7, b_7, b_7 };
1263 a_5 = _9 + { 0, 3, 6, 9 };
1264 because vector splat operation is usually more efficient
1265 than piecewise initialization of the vector. */
1268 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1270 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1271 tree lhs
= gimple_assign_lhs (stmt
);
1272 tree rhs
= gimple_assign_rhs1 (stmt
);
1273 tree type
= TREE_TYPE (rhs
);
1275 unsigned HOST_WIDE_INT nelts
;
1276 bool all_same
= true;
1277 constructor_elt
*elt
;
1279 tree base
= NULL_TREE
;
1282 if (!TYPE_VECTOR_SUBPARTS (type
).is_constant (&nelts
)
1284 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1286 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1287 if (op
== unknown_optab
1288 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1290 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1291 if (TREE_CODE (elt
->value
) != SSA_NAME
1292 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1296 tree this_base
= elt
->value
;
1297 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1299 for (j
= 0; j
< nelts
+ 1; j
++)
1301 g
= SSA_NAME_DEF_STMT (this_base
);
1302 if (is_gimple_assign (g
)
1303 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1304 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1305 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1306 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1307 this_base
= gimple_assign_rhs1 (g
);
1313 else if (this_base
!= base
)
1318 tree_vector_builder
cst (type
, nelts
, 1);
1319 for (i
= 0; i
< nelts
; i
++)
1321 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;
1322 tree elt
= build_zero_cst (TREE_TYPE (base
));
1323 while (this_base
!= base
)
1325 g
= SSA_NAME_DEF_STMT (this_base
);
1326 elt
= fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1327 elt
, gimple_assign_rhs2 (g
));
1328 if (elt
== NULL_TREE
1329 || TREE_CODE (elt
) != INTEGER_CST
1330 || TREE_OVERFLOW (elt
))
1332 this_base
= gimple_assign_rhs1 (g
);
1334 cst
.quick_push (elt
);
1336 for (i
= 0; i
< nelts
; i
++)
1337 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1338 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1339 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1340 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1342 gsi_replace (gsi
, g
, false);
1345 /* Return a type for the widest vector mode with the same element type as
1346 type ORIGINAL_VECTOR_TYPE, with at most the same number of elements as type
1347 ORIGINAL_VECTOR_TYPE and that is supported by the target for an operation
1348 with optab OP, or return NULL_TREE if none is found. */
1351 type_for_widest_vector_mode (tree original_vector_type
, optab op
)
1353 gcc_assert (VECTOR_TYPE_P (original_vector_type
));
1354 tree type
= TREE_TYPE (original_vector_type
);
1355 machine_mode inner_mode
= TYPE_MODE (type
);
1356 machine_mode best_mode
= VOIDmode
, mode
;
1357 poly_int64 best_nunits
= 0;
1359 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1360 mode
= MIN_MODE_VECTOR_FLOAT
;
1361 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1362 mode
= MIN_MODE_VECTOR_FRACT
;
1363 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1364 mode
= MIN_MODE_VECTOR_UFRACT
;
1365 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1366 mode
= MIN_MODE_VECTOR_ACCUM
;
1367 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1368 mode
= MIN_MODE_VECTOR_UACCUM
;
1369 else if (inner_mode
== BImode
)
1370 mode
= MIN_MODE_VECTOR_BOOL
;
1372 mode
= MIN_MODE_VECTOR_INT
;
1374 FOR_EACH_MODE_FROM (mode
, mode
)
1375 if (GET_MODE_INNER (mode
) == inner_mode
1376 && maybe_gt (GET_MODE_NUNITS (mode
), best_nunits
)
1377 && optab_handler (op
, mode
) != CODE_FOR_nothing
1378 && known_le (GET_MODE_NUNITS (mode
),
1379 TYPE_VECTOR_SUBPARTS (original_vector_type
)))
1380 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1382 if (best_mode
== VOIDmode
)
1385 return build_vector_type_for_mode (type
, best_mode
);
1389 /* Build a reference to the element of the vector VECT. Function
1390 returns either the element itself, either BIT_FIELD_REF, or an
1391 ARRAY_REF expression.
1393 GSI is required to insert temporary variables while building a
1394 refernece to the element of the vector VECT.
1396 PTMPVEC is a pointer to the temporary variable for caching
1397 purposes. In case when PTMPVEC is NULL new temporary variable
1400 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1402 tree vect_type
, vect_elt_type
;
1406 bool need_asgn
= true;
1407 unsigned int elements
;
1409 vect_type
= TREE_TYPE (vect
);
1410 vect_elt_type
= TREE_TYPE (vect_type
);
1411 elements
= nunits_for_known_piecewise_op (vect_type
);
1413 if (TREE_CODE (idx
) == INTEGER_CST
)
1415 unsigned HOST_WIDE_INT index
;
1417 /* Given that we're about to compute a binary modulus,
1418 we don't care about the high bits of the value. */
1419 index
= TREE_INT_CST_LOW (idx
);
1420 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1422 index
&= elements
- 1;
1423 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1426 /* When lowering a vector statement sequence do some easy
1427 simplification by looking through intermediate vector results. */
1428 if (TREE_CODE (vect
) == SSA_NAME
)
1430 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1431 if (is_gimple_assign (def_stmt
)
1432 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1433 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1434 vect
= gimple_assign_rhs1 (def_stmt
);
1437 if (TREE_CODE (vect
) == VECTOR_CST
)
1438 return VECTOR_CST_ELT (vect
, index
);
1439 else if (TREE_CODE (vect
) == CONSTRUCTOR
1440 && (CONSTRUCTOR_NELTS (vect
) == 0
1441 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1444 if (index
< CONSTRUCTOR_NELTS (vect
))
1445 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1446 return build_zero_cst (vect_elt_type
);
1450 tree size
= vector_element_bits_tree (vect_type
);
1451 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1453 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1458 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1460 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1469 TREE_ADDRESSABLE (tmpvec
) = 1;
1470 asgn
= gimple_build_assign (tmpvec
, vect
);
1471 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1474 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1475 return build4 (ARRAY_REF
, vect_elt_type
,
1476 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1477 idx
, NULL_TREE
, NULL_TREE
);
1480 /* Check if VEC_PERM_EXPR within the given setting is supported
1481 by hardware, or lower it piecewise.
1483 When VEC_PERM_EXPR has the same first and second operands:
1484 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1485 {v0[mask[0]], v0[mask[1]], ...}
1486 MASK and V0 must have the same number of elements.
1488 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1489 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1490 V0 and V1 must have the same type. MASK, V0, V1 must have the
1491 same number of arguments. */
1494 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1496 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1497 tree mask
= gimple_assign_rhs3 (stmt
);
1498 tree vec0
= gimple_assign_rhs1 (stmt
);
1499 tree vec1
= gimple_assign_rhs2 (stmt
);
1500 tree vect_type
= TREE_TYPE (vec0
);
1501 tree mask_type
= TREE_TYPE (mask
);
1502 tree vect_elt_type
= TREE_TYPE (vect_type
);
1503 tree mask_elt_type
= TREE_TYPE (mask_type
);
1504 unsigned HOST_WIDE_INT elements
;
1505 vec
<constructor_elt
, va_gc
> *v
;
1506 tree constr
, t
, si
, i_val
;
1507 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1508 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1509 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1512 if (!TYPE_VECTOR_SUBPARTS (vect_type
).is_constant (&elements
))
1515 if (TREE_CODE (mask
) == SSA_NAME
)
1517 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1518 if (is_gimple_assign (def_stmt
)
1519 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1520 mask
= gimple_assign_rhs1 (def_stmt
);
1523 vec_perm_builder sel_int
;
1525 if (TREE_CODE (mask
) == VECTOR_CST
1526 && tree_to_vec_perm_builder (&sel_int
, mask
))
1528 vec_perm_indices
indices (sel_int
, 2, elements
);
1529 machine_mode vmode
= TYPE_MODE (vect_type
);
1530 tree lhs_type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1531 machine_mode lhs_mode
= TYPE_MODE (lhs_type
);
1532 if (can_vec_perm_const_p (lhs_mode
, vmode
, indices
))
1534 gimple_assign_set_rhs3 (stmt
, mask
);
1538 /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1539 vector as VEC1 and a right element shift MASK. */
1540 if (optab_handler (vec_shr_optab
, TYPE_MODE (vect_type
))
1542 && TREE_CODE (vec1
) == VECTOR_CST
1543 && initializer_zerop (vec1
)
1544 && maybe_ne (indices
[0], 0)
1545 && known_lt (poly_uint64 (indices
[0]), elements
))
1547 bool ok_p
= indices
.series_p (0, 1, indices
[0], 1);
1550 for (i
= 1; i
< elements
; ++i
)
1552 poly_uint64 actual
= indices
[i
];
1553 poly_uint64 expected
= i
+ indices
[0];
1554 /* Indices into the second vector are all equivalent. */
1555 if (maybe_lt (actual
, elements
)
1556 ? maybe_ne (actual
, expected
)
1557 : maybe_lt (expected
, elements
))
1560 ok_p
= i
== elements
;
1564 gimple_assign_set_rhs3 (stmt
, mask
);
1569 /* And similarly vec_shl pattern. */
1570 if (optab_handler (vec_shl_optab
, TYPE_MODE (vect_type
))
1572 && TREE_CODE (vec0
) == VECTOR_CST
1573 && initializer_zerop (vec0
))
1575 unsigned int first
= 0;
1576 for (i
= 0; i
< elements
; ++i
)
1577 if (known_eq (poly_uint64 (indices
[i
]), elements
))
1579 if (i
== 0 || first
)
1584 ? maybe_ne (poly_uint64 (indices
[i
]),
1585 elements
+ i
- first
)
1586 : maybe_ge (poly_uint64 (indices
[i
]), elements
))
1588 if (first
&& i
== elements
)
1590 gimple_assign_set_rhs3 (stmt
, mask
);
1596 else if (can_vec_perm_var_p (TYPE_MODE (vect_type
)))
1599 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1600 warning_at (loc
, OPT_Wvector_operation_performance
,
1601 "vector shuffling operation will be expanded piecewise");
1603 vec_alloc (v
, elements
);
1604 bool constant_p
= true;
1605 for (i
= 0; i
< elements
; i
++)
1608 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1610 if (TREE_CODE (i_val
) == INTEGER_CST
)
1612 unsigned HOST_WIDE_INT index
;
1614 index
= TREE_INT_CST_LOW (i_val
);
1615 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1616 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1618 if (two_operand_p
&& (index
& elements
) != 0)
1619 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1621 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1623 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1624 true, GSI_SAME_STMT
);
1628 tree cond
= NULL_TREE
, v0_val
;
1632 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1633 build_int_cst (mask_elt_type
, elements
));
1634 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1635 true, GSI_SAME_STMT
);
1638 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1639 build_int_cst (mask_elt_type
, elements
- 1));
1640 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1641 true, GSI_SAME_STMT
);
1643 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1644 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1645 true, GSI_SAME_STMT
);
1651 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1652 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1653 true, GSI_SAME_STMT
);
1655 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1656 cond
, build_zero_cst (mask_elt_type
));
1657 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1658 cond
, v0_val
, v1_val
);
1659 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1660 true, GSI_SAME_STMT
);
1666 if (!CONSTANT_CLASS_P (t
))
1668 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1672 constr
= build_vector_from_ctor (vect_type
, v
);
1674 constr
= build_constructor (vect_type
, v
);
1675 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1676 update_stmt (gsi_stmt (*gsi
));
1679 /* If OP is a uniform vector return the element it is a splat from. */
1682 ssa_uniform_vector_p (tree op
)
1684 if (TREE_CODE (op
) == VECTOR_CST
1685 || TREE_CODE (op
) == VEC_DUPLICATE_EXPR
1686 || TREE_CODE (op
) == CONSTRUCTOR
)
1687 return uniform_vector_p (op
);
1688 if (TREE_CODE (op
) == SSA_NAME
)
1690 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1691 if (gimple_assign_single_p (def_stmt
))
1692 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1697 /* Return type in which CODE operation with optab OP can be
1701 get_compute_type (enum tree_code code
, optab op
, tree type
)
1703 /* For very wide vectors, try using a smaller vector mode. */
1704 tree compute_type
= type
;
1706 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1707 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1709 tree vector_compute_type
1710 = type_for_widest_vector_mode (type
, op
);
1711 if (vector_compute_type
!= NULL_TREE
1712 && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type
), 1U)
1713 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1714 != CODE_FOR_nothing
))
1715 compute_type
= vector_compute_type
;
1718 /* If we are breaking a BLKmode vector into smaller pieces,
1719 type_for_widest_vector_mode has already looked into the optab,
1720 so skip these checks. */
1721 if (compute_type
== type
)
1723 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1724 if (VECTOR_MODE_P (compute_mode
))
1727 && (optab_handler (op
, compute_mode
) != CODE_FOR_nothing
1728 || optab_libfunc (op
, compute_mode
)))
1729 return compute_type
;
1730 if (code
== MULT_HIGHPART_EXPR
1731 && can_mult_highpart_p (compute_mode
,
1732 TYPE_UNSIGNED (compute_type
)))
1733 return compute_type
;
1735 /* There is no operation in hardware, so fall back to scalars. */
1736 compute_type
= TREE_TYPE (type
);
1739 return compute_type
;
1743 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1744 tree bitpos
, tree bitsize
, enum tree_code code
,
1745 tree type ATTRIBUTE_UNUSED
)
1747 if (VECTOR_TYPE_P (TREE_TYPE (a
)))
1748 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1749 if (VECTOR_TYPE_P (TREE_TYPE (b
)))
1750 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1751 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1752 return gimplify_build3 (gsi
, code
, inner_type
, unshare_expr (cond
), a
, b
);
1755 /* Expand a vector COND_EXPR to scalars, piecewise. */
1757 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1759 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1760 tree lhs
= gimple_assign_lhs (stmt
);
1761 tree type
= TREE_TYPE (lhs
);
1762 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1763 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1764 gcc_assert (compute_mode
!= BLKmode
);
1765 tree rhs2
= gimple_assign_rhs2 (stmt
);
1766 tree rhs3
= gimple_assign_rhs3 (stmt
);
1769 /* If the compute mode is not a vector mode (hence we are not decomposing
1770 a BLKmode vector to smaller, hardware-supported vectors), we may want
1771 to expand the operations in parallel. */
1772 if (!VECTOR_MODE_P (compute_mode
))
1773 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1776 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1777 rhs2
, rhs3
, COND_EXPR
, false);
1778 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1779 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1782 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1783 way to do it is change expand_vector_operation and its callees to
1784 return a tree_code, RHS1 and RHS2 instead of a tree. */
1785 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1786 update_stmt (gsi_stmt (*gsi
));
1789 /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1790 lowering. If INNER_TYPE is not a vector type, this is a scalar
1794 do_vec_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1795 tree decl
, tree bitpos
, tree bitsize
,
1796 enum tree_code code
, tree type
)
1798 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1799 if (!VECTOR_TYPE_P (inner_type
))
1800 return gimplify_build1 (gsi
, code
, TREE_TYPE (type
), a
);
1801 if (code
== CALL_EXPR
)
1803 gimple
*g
= gimple_build_call (decl
, 1, a
);
1804 tree lhs
= make_ssa_name (TREE_TYPE (TREE_TYPE (decl
)));
1805 gimple_call_set_lhs (g
, lhs
);
1806 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1811 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1812 TYPE_VECTOR_SUBPARTS (inner_type
));
1813 return gimplify_build1 (gsi
, code
, outer_type
, a
);
1817 /* Similarly, but for narrowing conversion. */
1820 do_vec_narrow_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1821 tree
, tree bitpos
, tree
, enum tree_code code
,
1824 tree itype
= build_vector_type (TREE_TYPE (inner_type
),
1825 exact_div (TYPE_VECTOR_SUBPARTS (inner_type
),
1827 tree b
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
), bitpos
);
1828 tree c
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
),
1829 int_const_binop (PLUS_EXPR
, bitpos
,
1830 TYPE_SIZE (itype
)));
1831 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1832 TYPE_VECTOR_SUBPARTS (inner_type
));
1833 return gimplify_build2 (gsi
, code
, outer_type
, b
, c
);
1836 /* Expand VEC_CONVERT ifn call. */
1839 expand_vector_conversion (gimple_stmt_iterator
*gsi
)
1841 gimple
*stmt
= gsi_stmt (*gsi
);
1843 tree lhs
= gimple_call_lhs (stmt
);
1844 if (lhs
== NULL_TREE
)
1846 g
= gimple_build_nop ();
1847 gsi_replace (gsi
, g
, false);
1850 tree arg
= gimple_call_arg (stmt
, 0);
1851 tree ret_type
= TREE_TYPE (lhs
);
1852 tree arg_type
= TREE_TYPE (arg
);
1853 tree new_rhs
, compute_type
= TREE_TYPE (arg_type
);
1854 enum tree_code code
= NOP_EXPR
;
1855 enum tree_code code1
= ERROR_MARK
;
1856 enum { NARROW
, NONE
, WIDEN
} modifier
= NONE
;
1857 optab optab1
= unknown_optab
;
1859 gcc_checking_assert (VECTOR_TYPE_P (ret_type
) && VECTOR_TYPE_P (arg_type
));
1860 if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type
))
1861 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type
)))
1862 code
= FIX_TRUNC_EXPR
;
1863 else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type
))
1864 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type
)))
1866 unsigned int ret_elt_bits
= vector_element_bits (ret_type
);
1867 unsigned int arg_elt_bits
= vector_element_bits (arg_type
);
1868 if (ret_elt_bits
< arg_elt_bits
)
1870 else if (ret_elt_bits
> arg_elt_bits
)
1873 if (modifier
== NONE
&& (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
))
1875 if (supportable_convert_operation (code
, ret_type
, arg_type
, &code1
))
1877 g
= gimple_build_assign (lhs
, code1
, arg
);
1878 gsi_replace (gsi
, g
, false);
1881 /* Can't use get_compute_type here, as supportable_convert_operation
1882 doesn't necessarily use an optab and needs two arguments. */
1883 tree vec_compute_type
1884 = type_for_widest_vector_mode (arg_type
, mov_optab
);
1885 if (vec_compute_type
1886 && VECTOR_MODE_P (TYPE_MODE (vec_compute_type
)))
1888 unsigned HOST_WIDE_INT nelts
1889 = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type
));
1892 tree ret1_type
= build_vector_type (TREE_TYPE (ret_type
), nelts
);
1893 tree arg1_type
= build_vector_type (TREE_TYPE (arg_type
), nelts
);
1894 if (supportable_convert_operation (code
, ret1_type
, arg1_type
,
1897 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
,
1898 ret_type
, arg1_type
, arg
,
1899 NULL_TREE
, code1
, false);
1900 g
= gimple_build_assign (lhs
, new_rhs
);
1901 gsi_replace (gsi
, g
, false);
1908 else if (modifier
== NARROW
)
1913 code1
= VEC_PACK_TRUNC_EXPR
;
1914 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1916 case FIX_TRUNC_EXPR
:
1917 code1
= VEC_PACK_FIX_TRUNC_EXPR
;
1918 /* The signedness is determined from output operand. */
1919 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1922 code1
= VEC_PACK_FLOAT_EXPR
;
1923 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1930 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1931 enum insn_code icode1
;
1932 if (VECTOR_TYPE_P (compute_type
)
1933 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1934 != CODE_FOR_nothing
)
1935 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
))
1938 = build_vector_type (TREE_TYPE (ret_type
),
1939 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1940 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1942 if (compute_type
== arg_type
)
1944 new_rhs
= gimplify_build2 (gsi
, code1
, cretd_type
,
1945 arg
, build_zero_cst (arg_type
));
1946 new_rhs
= tree_vec_extract (gsi
, ret_type
, new_rhs
,
1947 TYPE_SIZE (ret_type
),
1949 g
= gimple_build_assign (lhs
, new_rhs
);
1950 gsi_replace (gsi
, g
, false);
1954 = build_vector_type (TREE_TYPE (compute_type
),
1955 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1956 if (TYPE_MAIN_VARIANT (dcompute_type
)
1957 == TYPE_MAIN_VARIANT (arg_type
))
1958 new_rhs
= do_vec_narrow_conversion (gsi
, dcompute_type
, arg
,
1959 NULL_TREE
, bitsize_int (0),
1963 new_rhs
= expand_vector_piecewise (gsi
,
1964 do_vec_narrow_conversion
,
1965 arg_type
, dcompute_type
,
1966 arg
, NULL_TREE
, code1
,
1968 g
= gimple_build_assign (lhs
, new_rhs
);
1969 gsi_replace (gsi
, g
, false);
1974 else if (modifier
== WIDEN
)
1976 enum tree_code code2
= ERROR_MARK
;
1977 optab optab2
= unknown_optab
;
1981 code1
= VEC_UNPACK_LO_EXPR
;
1982 code2
= VEC_UNPACK_HI_EXPR
;
1984 case FIX_TRUNC_EXPR
:
1985 code1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
1986 code2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
1989 code1
= VEC_UNPACK_FLOAT_LO_EXPR
;
1990 code2
= VEC_UNPACK_FLOAT_HI_EXPR
;
1995 if (BYTES_BIG_ENDIAN
)
1996 std::swap (code1
, code2
);
1998 if (code
== FIX_TRUNC_EXPR
)
2000 /* The signedness is determined from output operand. */
2001 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
2002 optab2
= optab_for_tree_code (code2
, ret_type
, optab_default
);
2006 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
2007 optab2
= optab_for_tree_code (code2
, arg_type
, optab_default
);
2010 if (optab1
&& optab2
)
2011 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
2013 enum insn_code icode1
, icode2
;
2014 if (VECTOR_TYPE_P (compute_type
)
2015 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
2016 != CODE_FOR_nothing
)
2017 && ((icode2
= optab_handler (optab2
, TYPE_MODE (compute_type
)))
2018 != CODE_FOR_nothing
)
2019 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
)
2020 && (insn_data
[icode1
].operand
[0].mode
2021 == insn_data
[icode2
].operand
[0].mode
))
2024 = exact_div (TYPE_VECTOR_SUBPARTS (compute_type
), 2);
2025 tree cretd_type
= build_vector_type (TREE_TYPE (ret_type
), nunits
);
2026 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
2028 vec
<constructor_elt
, va_gc
> *v
;
2029 tree part_width
= TYPE_SIZE (compute_type
);
2030 tree index
= bitsize_int (0);
2031 int nunits
= nunits_for_known_piecewise_op (arg_type
);
2032 int delta
= tree_to_uhwi (part_width
) / arg_elt_bits
;
2034 location_t loc
= gimple_location (gsi_stmt (*gsi
));
2036 if (compute_type
!= arg_type
)
2038 if (!warning_suppressed_p (gsi_stmt (*gsi
),
2039 OPT_Wvector_operation_performance
))
2040 warning_at (loc
, OPT_Wvector_operation_performance
,
2041 "vector operation will be expanded piecewise");
2049 vec_alloc (v
, (nunits
+ delta
- 1) / delta
* 2);
2050 bool constant_p
= true;
2051 for (i
= 0; i
< nunits
;
2052 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
,
2056 if (compute_type
!= arg_type
)
2057 a
= tree_vec_extract (gsi
, compute_type
, a
, part_width
,
2059 tree result
= gimplify_build1 (gsi
, code1
, cretd_type
, a
);
2060 constructor_elt ce
= { NULL_TREE
, result
};
2061 if (!CONSTANT_CLASS_P (ce
.value
))
2064 ce
.value
= gimplify_build1 (gsi
, code2
, cretd_type
, a
);
2065 if (!CONSTANT_CLASS_P (ce
.value
))
2071 new_rhs
= build_vector_from_ctor (ret_type
, v
);
2073 new_rhs
= build_constructor (ret_type
, v
);
2074 g
= gimple_build_assign (lhs
, new_rhs
);
2075 gsi_replace (gsi
, g
, false);
2081 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
, arg_type
,
2082 TREE_TYPE (arg_type
), arg
,
2083 NULL_TREE
, code
, false, ret_type
);
2084 g
= gimple_build_assign (lhs
, new_rhs
);
2085 gsi_replace (gsi
, g
, false);
2088 /* Process one statement. If we identify a vector operation, expand it. */
2091 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
,
2092 bitmap dce_ssa_names
)
2094 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
2095 enum tree_code code
;
2096 optab op
= unknown_optab
;
2097 enum gimple_rhs_class rhs_class
;
2100 /* Only consider code == GIMPLE_ASSIGN. */
2101 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
2104 if (gimple_call_internal_p (gsi_stmt (*gsi
), IFN_VEC_CONVERT
))
2105 expand_vector_conversion (gsi
);
2109 code
= gimple_assign_rhs_code (stmt
);
2110 rhs_class
= get_gimple_rhs_class (code
);
2111 lhs
= gimple_assign_lhs (stmt
);
2113 if (code
== VEC_PERM_EXPR
)
2115 lower_vec_perm (gsi
);
2119 if (code
== VEC_COND_EXPR
)
2121 expand_vector_condition (gsi
, dce_ssa_names
);
2125 if (code
== COND_EXPR
2126 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
2127 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
2129 expand_vector_scalar_condition (gsi
);
2133 if (code
== CONSTRUCTOR
2134 && TREE_CODE (lhs
) == SSA_NAME
2135 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
2136 && !gimple_clobber_p (stmt
)
2139 optimize_vector_constructor (gsi
);
2143 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
2146 rhs1
= gimple_assign_rhs1 (stmt
);
2147 if (rhs_class
== GIMPLE_BINARY_RHS
)
2148 rhs2
= gimple_assign_rhs2 (stmt
);
2150 type
= TREE_TYPE (lhs
);
2151 if (!VECTOR_TYPE_P (type
)
2152 || !VECTOR_TYPE_P (TREE_TYPE (rhs1
)))
2155 /* A scalar operation pretending to be a vector one. */
2156 if (VECTOR_BOOLEAN_TYPE_P (type
)
2157 && !VECTOR_MODE_P (TYPE_MODE (type
))
2158 && TYPE_MODE (type
) != BLKmode
2159 && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) != tcc_comparison
2160 || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
))
2161 && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1
)))
2162 && TYPE_MODE (TREE_TYPE (rhs1
)) != BLKmode
)))
2165 /* If the vector operation is operating on all same vector elements
2166 implement it with a scalar operation and a splat if the target
2167 supports the scalar operation. */
2168 tree srhs1
, srhs2
= NULL_TREE
;
2169 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
2170 && (rhs2
== NULL_TREE
2171 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
2173 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2174 /* As we query direct optabs restrict to non-convert operations. */
2175 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
2177 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
2178 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
2179 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
2181 tree stype
= TREE_TYPE (TREE_TYPE (lhs
));
2182 tree slhs
= (rhs2
!= NULL_TREE
)
2183 ? gimplify_build2 (gsi
, code
, stype
, srhs1
, srhs2
)
2184 : gimplify_build1 (gsi
, code
, stype
, srhs1
);
2185 gimple_assign_set_rhs_from_tree (gsi
,
2186 build_vector_from_val (type
, slhs
));
2192 if (CONVERT_EXPR_CODE_P (code
)
2193 || code
== FLOAT_EXPR
2194 || code
== FIX_TRUNC_EXPR
2195 || code
== VIEW_CONVERT_EXPR
)
2198 /* The signedness is determined from input argument. */
2199 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
2200 || code
== VEC_UNPACK_FLOAT_LO_EXPR
2201 || code
== VEC_PACK_FLOAT_EXPR
)
2203 /* We do not know how to scalarize those. */
2207 /* For widening/narrowing vector operations, the relevant type is of the
2208 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2209 calculated in the same way above. */
2210 if (code
== WIDEN_SUM_EXPR
2211 || code
== VEC_WIDEN_MULT_HI_EXPR
2212 || code
== VEC_WIDEN_MULT_LO_EXPR
2213 || code
== VEC_WIDEN_MULT_EVEN_EXPR
2214 || code
== VEC_WIDEN_MULT_ODD_EXPR
2215 || code
== VEC_UNPACK_HI_EXPR
2216 || code
== VEC_UNPACK_LO_EXPR
2217 || code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
2218 || code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
2219 || code
== VEC_PACK_TRUNC_EXPR
2220 || code
== VEC_PACK_SAT_EXPR
2221 || code
== VEC_PACK_FIX_TRUNC_EXPR
2222 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
2223 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
2225 /* We do not know how to scalarize those. */
2229 /* Choose between vector shift/rotate by vector and vector shift/rotate by
2231 if (code
== LSHIFT_EXPR
2232 || code
== RSHIFT_EXPR
2233 || code
== LROTATE_EXPR
2234 || code
== RROTATE_EXPR
)
2238 /* Check whether we have vector <op> {x,x,x,x} where x
2239 could be a scalar variable or a constant. Transform
2240 vector <op> {x,x,x,x} ==> vector <op> scalar. */
2241 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2245 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2247 gimple_assign_set_rhs2 (stmt
, first
);
2253 opv
= optab_for_tree_code (code
, type
, optab_vector
);
2254 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2258 op
= optab_for_tree_code (code
, type
, optab_scalar
);
2260 compute_type
= get_compute_type (code
, op
, type
);
2261 if (compute_type
== type
)
2263 /* The rtl expander will expand vector/scalar as vector/vector
2264 if necessary. Pick one with wider vector type. */
2265 tree compute_vtype
= get_compute_type (code
, opv
, type
);
2266 if (subparts_gt (compute_vtype
, compute_type
))
2268 compute_type
= compute_vtype
;
2273 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
2275 if (compute_type
== NULL_TREE
)
2276 compute_type
= get_compute_type (code
, op
, type
);
2277 if (compute_type
== type
)
2279 /* Before splitting vector rotates into scalar rotates,
2280 see if we can't use vector shifts and BIT_IOR_EXPR
2281 instead. For vector by vector rotates we'd also
2282 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2283 for now, fold doesn't seem to create such rotates anyway. */
2284 if (compute_type
== TREE_TYPE (type
)
2285 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2287 optab oplv
= vashl_optab
, opl
= ashl_optab
;
2288 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
2289 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
2290 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
2291 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
2292 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
2293 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
2294 /* The rtl expander will expand vector/scalar as vector/vector
2295 if necessary. Pick one with wider vector type. */
2296 if (subparts_gt (compute_lvtype
, compute_ltype
))
2298 compute_ltype
= compute_lvtype
;
2301 if (subparts_gt (compute_rvtype
, compute_rtype
))
2303 compute_rtype
= compute_rvtype
;
2306 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2308 compute_type
= compute_ltype
;
2309 if (subparts_gt (compute_type
, compute_rtype
))
2310 compute_type
= compute_rtype
;
2311 if (subparts_gt (compute_type
, compute_otype
))
2312 compute_type
= compute_otype
;
2313 /* Verify all 3 operations can be performed in that type. */
2314 if (compute_type
!= TREE_TYPE (type
))
2316 if (optab_handler (opl
, TYPE_MODE (compute_type
))
2318 || optab_handler (opr
, TYPE_MODE (compute_type
))
2320 || optab_handler (opo
, TYPE_MODE (compute_type
))
2321 == CODE_FOR_nothing
)
2322 compute_type
= TREE_TYPE (type
);
2328 op
= optab_for_tree_code (code
, type
, optab_default
);
2330 /* Optabs will try converting a negation into a subtraction, so
2331 look for it as well. TODO: negation of floating-point vectors
2332 might be turned into an exclusive OR toggling the sign bit. */
2333 if (op
== unknown_optab
2334 && code
== NEGATE_EXPR
2335 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
2336 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
2338 if (compute_type
== NULL_TREE
)
2339 compute_type
= get_compute_type (code
, op
, type
);
2340 if (compute_type
== type
)
2343 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
,
2346 /* Leave expression untouched for later expansion. */
2347 if (new_rhs
== NULL_TREE
)
2350 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
2351 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
2354 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2355 way to do it is change expand_vector_operation and its callees to
2356 return a tree_code, RHS1 and RHS2 instead of a tree. */
2357 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
2358 update_stmt (gsi_stmt (*gsi
));
2361 /* Use this to lower vector operations introduced by the vectorizer,
2362 if it may need the bit-twiddling tricks implemented in this file. */
2365 expand_vector_operations (void)
2367 gimple_stmt_iterator gsi
;
2369 bool cfg_changed
= false;
2371 auto_bitmap dce_ssa_names
;
2373 FOR_EACH_BB_FN (bb
, cfun
)
2375 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2377 expand_vector_operations_1 (&gsi
, dce_ssa_names
);
2378 /* ??? If we do not cleanup EH then we will ICE in
2379 verification. But in reality we have created wrong-code
2380 as we did not properly transition EH info and edges to
2381 the piecewise computations. */
2382 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
2383 && gimple_purge_dead_eh_edges (bb
))
2385 /* If a .LOOP_DIST_ALIAS call prevailed loops got elided
2386 before vectorization got a chance to get at them. Simply
2387 fold as if loop distribution wasn't performed. */
2388 if (gimple_call_internal_p (gsi_stmt (gsi
), IFN_LOOP_DIST_ALIAS
))
2390 fold_loop_internal_call (gsi_stmt (gsi
), boolean_false_node
);
2396 simple_dce_from_worklist (dce_ssa_names
);
2398 return cfg_changed
? TODO_cleanup_cfg
: 0;
2403 const pass_data pass_data_lower_vector
=
2405 GIMPLE_PASS
, /* type */
2406 "veclower", /* name */
2407 OPTGROUP_VEC
, /* optinfo_flags */
2408 TV_NONE
, /* tv_id */
2409 PROP_cfg
, /* properties_required */
2410 PROP_gimple_lvec
, /* properties_provided */
2411 0, /* properties_destroyed */
2412 0, /* todo_flags_start */
2413 TODO_update_ssa
, /* todo_flags_finish */
2416 class pass_lower_vector
: public gimple_opt_pass
2419 pass_lower_vector (gcc::context
*ctxt
)
2420 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
2423 /* opt_pass methods: */
2424 bool gate (function
*fun
) final override
2426 return !(fun
->curr_properties
& PROP_gimple_lvec
);
2429 unsigned int execute (function
*) final override
2431 return expand_vector_operations ();
2434 }; // class pass_lower_vector
2439 make_pass_lower_vector (gcc::context
*ctxt
)
2441 return new pass_lower_vector (ctxt
);
2446 const pass_data pass_data_lower_vector_ssa
=
2448 GIMPLE_PASS
, /* type */
2449 "veclower2", /* name */
2450 OPTGROUP_VEC
, /* optinfo_flags */
2451 TV_NONE
, /* tv_id */
2452 PROP_cfg
, /* properties_required */
2453 PROP_gimple_lvec
, /* properties_provided */
2454 0, /* properties_destroyed */
2455 0, /* todo_flags_start */
2457 | TODO_cleanup_cfg
), /* todo_flags_finish */
2460 class pass_lower_vector_ssa
: public gimple_opt_pass
2463 pass_lower_vector_ssa (gcc::context
*ctxt
)
2464 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
2467 /* opt_pass methods: */
2468 opt_pass
* clone () final override
2470 return new pass_lower_vector_ssa (m_ctxt
);
2472 unsigned int execute (function
*) final override
2474 return expand_vector_operations ();
2477 }; // class pass_lower_vector_ssa
2482 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
2484 return new pass_lower_vector_ssa (ctxt
);
2487 #include "gt-tree-vect-generic.h"