1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2020 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "diagnostic.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
34 #include "langhooks.h"
36 #include "gimple-iterator.h"
37 #include "gimplify-me.h"
40 #include "tree-vector-builder.h"
41 #include "vec-perm-indices.h"
42 #include "insn-config.h"
43 #include "recog.h" /* FIXME: for insn_data */
46 static void expand_vector_operations_1 (gimple_stmt_iterator
*);
48 /* Return the number of elements in a vector type TYPE that we have
49 already decided needs to be expanded piecewise. We don't support
50 this kind of expansion for variable-length vectors, since we should
51 always check for target support before introducing uses of those. */
53 nunits_for_known_piecewise_op (const_tree type
)
55 return TYPE_VECTOR_SUBPARTS (type
).to_constant ();
58 /* Return true if TYPE1 has more elements than TYPE2, where either
59 type may be a vector or a scalar. */
62 subparts_gt (tree type1
, tree type2
)
64 poly_uint64 n1
= VECTOR_TYPE_P (type1
) ? TYPE_VECTOR_SUBPARTS (type1
) : 1;
65 poly_uint64 n2
= VECTOR_TYPE_P (type2
) ? TYPE_VECTOR_SUBPARTS (type2
) : 1;
66 return known_gt (n1
, n2
);
69 /* Build a constant of type TYPE, made of VALUE's bits replicated
70 every WIDTH bits to fit TYPE's precision. */
72 build_replicated_const (tree type
, unsigned int width
, HOST_WIDE_INT value
)
74 int n
= (TYPE_PRECISION (type
) + HOST_BITS_PER_WIDE_INT
- 1)
75 / HOST_BITS_PER_WIDE_INT
;
76 unsigned HOST_WIDE_INT low
, mask
;
77 HOST_WIDE_INT a
[WIDE_INT_MAX_ELTS
];
80 gcc_assert (n
&& n
<= WIDE_INT_MAX_ELTS
);
82 if (width
== HOST_BITS_PER_WIDE_INT
)
86 mask
= ((HOST_WIDE_INT
)1 << width
) - 1;
87 low
= (unsigned HOST_WIDE_INT
) ~0 / mask
* (value
& mask
);
90 for (i
= 0; i
< n
; i
++)
93 gcc_assert (TYPE_PRECISION (type
) <= MAX_BITSIZE_MODE_ANY_INT
);
94 return wide_int_to_tree
95 (type
, wide_int::from_array (a
, n
, TYPE_PRECISION (type
)));
98 static GTY(()) tree vector_inner_type
;
99 static GTY(()) tree vector_last_type
;
100 static GTY(()) int vector_last_nunits
;
102 /* Return a suitable vector types made of SUBPARTS units each of mode
103 "word_mode" (the global variable). */
105 build_word_mode_vector_type (int nunits
)
107 if (!vector_inner_type
)
108 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
109 else if (vector_last_nunits
== nunits
)
111 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
112 return vector_last_type
;
115 vector_last_nunits
= nunits
;
116 vector_last_type
= build_vector_type (vector_inner_type
, nunits
);
117 return vector_last_type
;
120 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
121 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
125 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
126 tree t
, tree bitsize
, tree bitpos
)
128 if (TREE_CODE (t
) == SSA_NAME
)
130 gimple
*def_stmt
= SSA_NAME_DEF_STMT (t
);
131 if (is_gimple_assign (def_stmt
)
132 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
134 && gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
)))
135 t
= gimple_assign_rhs1 (def_stmt
);
139 if (TREE_CODE (type
) == BOOLEAN_TYPE
)
142 = build_nonstandard_integer_type (tree_to_uhwi (bitsize
), 0);
143 tree field
= gimplify_build3 (gsi
, BIT_FIELD_REF
, itype
, t
,
145 return gimplify_build2 (gsi
, NE_EXPR
, type
, field
,
146 build_zero_cst (itype
));
149 return gimplify_build3 (gsi
, BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
152 return gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
156 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
157 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
158 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
160 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
161 return gimplify_build1 (gsi
, code
, inner_type
, a
);
165 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
166 tree bitpos
, tree bitsize
, enum tree_code code
,
167 tree type ATTRIBUTE_UNUSED
)
169 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
170 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
171 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
172 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
173 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
176 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
178 INNER_TYPE is the type of A and B elements
180 returned expression is of signed integer type with the
181 size equal to the size of INNER_TYPE. */
183 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
184 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
186 tree stype
= TREE_TYPE (type
);
187 tree cst_false
= build_zero_cst (stype
);
188 tree cst_true
= build_all_ones_cst (stype
);
191 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
192 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
194 cmp
= build2 (code
, boolean_type_node
, a
, b
);
195 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
198 /* Expand vector addition to scalars. This does bit twiddling
199 in order to increase parallelism:
201 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
204 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
205 (a ^ ~b) & 0x80808080
207 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
209 This optimization should be done only if 4 vector items or more
212 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
213 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
214 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
216 unsigned int width
= vector_element_bits (TREE_TYPE (a
));
217 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
218 unsigned HOST_WIDE_INT max
;
219 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
221 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
222 low_bits
= build_replicated_const (word_type
, width
, max
>> 1);
223 high_bits
= build_replicated_const (word_type
, width
, max
& ~(max
>> 1));
225 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
226 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
228 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
229 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
230 if (code
== PLUS_EXPR
)
231 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
234 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
235 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
238 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
239 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
240 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
244 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
245 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
246 tree bitsize ATTRIBUTE_UNUSED
,
247 enum tree_code code ATTRIBUTE_UNUSED
,
248 tree type ATTRIBUTE_UNUSED
)
250 unsigned int width
= vector_element_bits (TREE_TYPE (b
));
251 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
253 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
255 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
256 low_bits
= build_replicated_const (word_type
, width
, max
>> 1);
257 high_bits
= build_replicated_const (word_type
, width
, max
& ~(max
>> 1));
259 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
261 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
262 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
263 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
264 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
265 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
268 /* Expand a vector operation to scalars, by using many operations
269 whose type is the vector type's inner type. */
271 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
272 tree type
, tree inner_type
,
273 tree a
, tree b
, enum tree_code code
,
274 tree ret_type
= NULL_TREE
)
276 vec
<constructor_elt
, va_gc
> *v
;
277 tree part_width
= TYPE_SIZE (inner_type
);
278 tree index
= bitsize_int (0);
279 int nunits
= nunits_for_known_piecewise_op (type
);
280 int delta
= tree_to_uhwi (part_width
) / vector_element_bits (type
);
282 location_t loc
= gimple_location (gsi_stmt (*gsi
));
285 || types_compatible_p (gimple_expr_type (gsi_stmt (*gsi
)), type
))
286 warning_at (loc
, OPT_Wvector_operation_performance
,
287 "vector operation will be expanded piecewise");
289 warning_at (loc
, OPT_Wvector_operation_performance
,
290 "vector operation will be expanded in parallel");
294 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
295 for (i
= 0; i
< nunits
;
296 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
298 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
,
300 constructor_elt ce
= {NULL_TREE
, result
};
304 return build_constructor (ret_type
, v
);
307 /* Expand a vector operation to scalars with the freedom to use
308 a scalar integer type, or to use a different size for the items
309 in the vector type. */
311 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
312 tree a
, tree b
, enum tree_code code
)
314 tree result
, compute_type
;
315 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
316 location_t loc
= gimple_location (gsi_stmt (*gsi
));
318 /* We have three strategies. If the type is already correct, just do
319 the operation an element at a time. Else, if the vector is wider than
320 one word, do it a word at a time; finally, if the vector is smaller
321 than one word, do it as a scalar. */
322 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
323 return expand_vector_piecewise (gsi
, f
,
324 type
, TREE_TYPE (type
),
326 else if (n_words
> 1)
328 tree word_type
= build_word_mode_vector_type (n_words
);
329 result
= expand_vector_piecewise (gsi
, f
,
330 word_type
, TREE_TYPE (word_type
),
332 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
337 /* Use a single scalar operation with a mode no wider than word_mode. */
339 = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), 0).require ();
340 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
341 result
= f (gsi
, compute_type
, a
, b
, NULL_TREE
, NULL_TREE
, code
, type
);
342 warning_at (loc
, OPT_Wvector_operation_performance
,
343 "vector operation will be expanded with a "
344 "single scalar operation");
350 /* Expand a vector operation to scalars; for integer types we can use
351 special bit twiddling tricks to do the sums a word at a time, using
352 function F_PARALLEL instead of F. These tricks are done only if
353 they can process at least four items, that is, only if the vector
354 holds at least four items and if a word can hold four items. */
356 expand_vector_addition (gimple_stmt_iterator
*gsi
,
357 elem_op_func f
, elem_op_func f_parallel
,
358 tree type
, tree a
, tree b
, enum tree_code code
)
360 int parts_per_word
= BITS_PER_WORD
/ vector_element_bits (type
);
362 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
363 && parts_per_word
>= 4
364 && nunits_for_known_piecewise_op (type
) >= 4)
365 return expand_vector_parallel (gsi
, f_parallel
,
368 return expand_vector_piecewise (gsi
, f
,
369 type
, TREE_TYPE (type
),
373 /* Try to expand vector comparison expression OP0 CODE OP1 by
374 querying optab if the following expression:
375 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
378 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
379 tree op1
, enum tree_code code
)
382 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
, code
)
383 && !expand_vec_cond_expr_p (type
, TREE_TYPE (op0
), code
))
385 if (VECTOR_BOOLEAN_TYPE_P (type
)
386 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
387 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
388 TYPE_VECTOR_SUBPARTS (type
)
389 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
390 (TREE_TYPE (type
)))))
392 tree inner_type
= TREE_TYPE (TREE_TYPE (op0
));
393 tree part_width
= vector_element_bits_tree (TREE_TYPE (op0
));
394 tree index
= bitsize_int (0);
395 int nunits
= nunits_for_known_piecewise_op (TREE_TYPE (op0
));
396 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (type
));
397 tree ret_type
= build_nonstandard_integer_type (prec
, 1);
398 tree ret_inner_type
= boolean_type_node
;
400 location_t loc
= gimple_location (gsi_stmt (*gsi
));
401 t
= build_zero_cst (ret_type
);
403 if (TYPE_PRECISION (ret_inner_type
) != 1)
404 ret_inner_type
= build_nonstandard_integer_type (1, 1);
405 warning_at (loc
, OPT_Wvector_operation_performance
,
406 "vector operation will be expanded piecewise");
407 for (i
= 0; i
< nunits
;
408 i
++, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
410 tree a
= tree_vec_extract (gsi
, inner_type
, op0
, part_width
,
412 tree b
= tree_vec_extract (gsi
, inner_type
, op1
, part_width
,
414 tree result
= gimplify_build2 (gsi
, code
, ret_inner_type
, a
, b
);
415 t
= gimplify_build3 (gsi
, BIT_INSERT_EXPR
, ret_type
, t
, result
,
418 t
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
421 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
422 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
,
431 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
432 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
433 the result if successful, otherwise return NULL_TREE. */
435 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
438 unsigned int i
, nunits
= nunits_for_known_piecewise_op (type
);
439 bool scalar_shift
= true;
441 for (i
= 1; i
< nunits
; i
++)
443 if (shiftcnts
[i
] != shiftcnts
[0])
444 scalar_shift
= false;
447 if (scalar_shift
&& shiftcnts
[0] == 0)
452 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
453 if (op
!= unknown_optab
454 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
455 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
456 build_int_cst (NULL_TREE
, shiftcnts
[0]));
459 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
460 if (op
!= unknown_optab
461 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
463 tree_vector_builder
vec (type
, nunits
, 1);
464 for (i
= 0; i
< nunits
; i
++)
465 vec
.quick_push (build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]));
466 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
, vec
.build ());
472 /* Try to expand integer vector division by constant using
473 widening multiply, shifts and additions. */
475 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
476 tree op1
, enum tree_code code
)
478 bool use_pow2
= true;
479 bool has_vector_shift
= true;
480 bool use_abs_op1
= false;
481 int mode
= -1, this_mode
;
482 int pre_shift
= -1, post_shift
;
483 unsigned int nunits
= nunits_for_known_piecewise_op (type
);
484 int *shifts
= XALLOCAVEC (int, nunits
* 4);
485 int *pre_shifts
= shifts
+ nunits
;
486 int *post_shifts
= pre_shifts
+ nunits
;
487 int *shift_temps
= post_shifts
+ nunits
;
488 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
489 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
492 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
493 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
494 tree cur_op
, mulcst
, tem
;
497 if (prec
> HOST_BITS_PER_WIDE_INT
)
500 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
501 if (op
== unknown_optab
502 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
503 has_vector_shift
= false;
505 /* Analysis phase. Determine if all op1 elements are either power
506 of two and it is possible to expand it using shifts (or for remainder
507 using masking). Additionally compute the multiplicative constants
508 and pre and post shifts if the division is to be expanded using
509 widening or high part multiplication plus shifts. */
510 for (i
= 0; i
< nunits
; i
++)
512 tree cst
= VECTOR_CST_ELT (op1
, i
);
513 unsigned HOST_WIDE_INT ml
;
515 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
521 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
525 shifts
[i
] = tree_log2 (cst
);
526 if (shifts
[i
] != shifts
[0]
527 && code
== TRUNC_DIV_EXPR
528 && !has_vector_shift
)
533 if (sign_p
== UNSIGNED
)
535 unsigned HOST_WIDE_INT mh
;
536 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
538 if (d
>= (HOST_WIDE_INT_1U
<< (prec
- 1)))
539 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
548 /* Find a suitable multiplier and right shift count
549 instead of multiplying with D. */
550 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
552 /* If the suggested multiplier is more than SIZE bits, we can
553 do better for even divisors, using an initial right shift. */
554 if ((mh
!= 0 && (d
& 1) == 0)
555 || (!has_vector_shift
&& pre_shift
!= -1))
557 if (has_vector_shift
)
558 pre_shift
= ctz_or_zero (d
);
559 else if (pre_shift
== -1)
562 for (j
= 0; j
< nunits
; j
++)
564 tree cst2
= VECTOR_CST_ELT (op1
, j
);
565 unsigned HOST_WIDE_INT d2
;
568 if (!tree_fits_uhwi_p (cst2
))
570 d2
= tree_to_uhwi (cst2
) & mask
;
573 this_pre_shift
= floor_log2 (d2
& -d2
);
574 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
575 pre_shift
= this_pre_shift
;
577 if (i
!= 0 && pre_shift
!= 0)
587 if ((d
>> pre_shift
) <= 1)
592 mh
= choose_multiplier (d
>> pre_shift
, prec
,
594 &ml
, &post_shift
, &dummy_int
);
596 pre_shifts
[i
] = pre_shift
;
606 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
607 unsigned HOST_WIDE_INT abs_d
;
612 /* Since d might be INT_MIN, we have to cast to
613 unsigned HOST_WIDE_INT before negating to avoid
614 undefined signed overflow. */
616 ? (unsigned HOST_WIDE_INT
) d
617 : - (unsigned HOST_WIDE_INT
) d
);
619 /* n rem d = n rem -d */
620 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
625 if (abs_d
== HOST_WIDE_INT_1U
<< (prec
- 1))
627 /* This case is not handled correctly below. */
637 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
638 &post_shift
, &dummy_int
);
639 if (ml
>= HOST_WIDE_INT_1U
<< (prec
- 1))
641 this_mode
= 4 + (d
< 0);
642 ml
|= HOST_WIDE_INT_M1U
<< (prec
- 1);
645 this_mode
= 2 + (d
< 0);
648 post_shifts
[i
] = post_shift
;
649 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
650 || post_shift
>= prec
651 || pre_shifts
[i
] >= prec
)
656 else if (mode
!= this_mode
)
662 tree addend
= NULL_TREE
;
663 if (sign_p
== SIGNED
)
667 /* Both division and remainder sequences need
668 op0 < 0 ? mask : 0 computed. It can be either computed as
669 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
670 if none of the shifts is 0, or as the conditional. */
671 for (i
= 0; i
< nunits
; i
++)
675 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
677 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
679 for (i
= 0; i
< nunits
; i
++)
680 shift_temps
[i
] = prec
- 1;
681 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
682 if (cur_op
!= NULL_TREE
)
684 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
686 for (i
= 0; i
< nunits
; i
++)
687 shift_temps
[i
] = prec
- shifts
[i
];
688 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
689 if (cur_op
!= NULL_TREE
)
690 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
694 if (addend
== NULL_TREE
695 && expand_vec_cond_expr_p (type
, type
, LT_EXPR
))
697 tree zero
, cst
, cond
, mask_type
;
700 mask_type
= truth_type_for (type
);
701 zero
= build_zero_cst (type
);
702 cond
= build2 (LT_EXPR
, mask_type
, op0
, zero
);
703 tree_vector_builder
vec (type
, nunits
, 1);
704 for (i
= 0; i
< nunits
; i
++)
705 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
709 addend
= make_ssa_name (type
);
710 stmt
= gimple_build_assign (addend
, VEC_COND_EXPR
, cond
,
712 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
715 if (code
== TRUNC_DIV_EXPR
)
717 if (sign_p
== UNSIGNED
)
719 /* q = op0 >> shift; */
720 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
721 if (cur_op
!= NULL_TREE
)
724 else if (addend
!= NULL_TREE
)
726 /* t1 = op0 + addend;
728 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
729 if (op
!= unknown_optab
730 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
732 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
733 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
734 if (cur_op
!= NULL_TREE
)
742 tree_vector_builder
vec (type
, nunits
, 1);
743 for (i
= 0; i
< nunits
; i
++)
744 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
748 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
749 if (op
!= unknown_optab
750 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
752 if (sign_p
== UNSIGNED
)
753 /* r = op0 & mask; */
754 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
755 else if (addend
!= NULL_TREE
)
757 /* t1 = op0 + addend;
760 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
761 if (op
!= unknown_optab
762 && optab_handler (op
, TYPE_MODE (type
))
765 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
767 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
769 op
= optab_for_tree_code (MINUS_EXPR
, type
,
771 if (op
!= unknown_optab
772 && optab_handler (op
, TYPE_MODE (type
))
774 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
782 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
785 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
793 gcc_assert (sign_p
== UNSIGNED
);
794 /* t1 = oprnd0 >> pre_shift;
796 q = t2 >> post_shift; */
797 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
798 if (cur_op
== NULL_TREE
)
802 gcc_assert (sign_p
== UNSIGNED
);
803 for (i
= 0; i
< nunits
; i
++)
813 gcc_assert (sign_p
== SIGNED
);
814 for (i
= 0; i
< nunits
; i
++)
815 shift_temps
[i
] = prec
- 1;
821 tree_vector_builder
vec (type
, nunits
, 1);
822 for (i
= 0; i
< nunits
; i
++)
823 vec
.quick_push (build_int_cst (TREE_TYPE (type
), mulc
[i
]));
824 mulcst
= vec
.build ();
826 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
831 /* t1 = oprnd0 >> pre_shift;
833 q = t2 >> post_shift; */
834 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
837 /* t1 = oprnd0 h* ml;
841 q = t4 >> (post_shift - 1); */
842 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
843 if (op
== unknown_optab
844 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
846 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
847 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
848 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
849 if (op
== unknown_optab
850 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
852 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
853 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
854 if (cur_op
== NULL_TREE
)
861 /* t1 = oprnd0 h* ml;
862 t2 = t1; [ iff (mode & 2) != 0 ]
863 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
864 t3 = t2 >> post_shift;
865 t4 = oprnd0 >> (prec - 1);
866 q = t3 - t4; [ iff (mode & 1) == 0 ]
867 q = t4 - t3; [ iff (mode & 1) != 0 ] */
870 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
871 if (op
== unknown_optab
872 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
874 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
876 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
877 if (cur_op
== NULL_TREE
)
879 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
880 if (tem
== NULL_TREE
)
882 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
883 if (op
== unknown_optab
884 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
887 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
889 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
895 if (code
== TRUNC_DIV_EXPR
)
898 /* We divided. Now finish by:
901 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
902 if (op
== unknown_optab
903 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
907 tree_vector_builder elts
;
908 if (!elts
.new_unary_operation (type
, op1
, false))
910 unsigned int count
= elts
.encoded_nelts ();
911 for (unsigned int i
= 0; i
< count
; ++i
)
913 tree elem1
= VECTOR_CST_ELT (op1
, i
);
915 tree elt
= const_unop (ABS_EXPR
, TREE_TYPE (elem1
), elem1
);
916 if (elt
== NULL_TREE
)
918 elts
.quick_push (elt
);
922 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
923 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
924 if (op
== unknown_optab
925 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
927 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
930 /* Expand a vector condition to scalars, by using many conditions
931 on the vector's elements. */
933 expand_vector_condition (gimple_stmt_iterator
*gsi
)
935 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
936 tree type
= gimple_expr_type (stmt
);
937 tree a
= gimple_assign_rhs1 (stmt
);
940 bool a_is_comparison
= false;
941 bool a_is_scalar_bitmask
= false;
942 tree b
= gimple_assign_rhs2 (stmt
);
943 tree c
= gimple_assign_rhs3 (stmt
);
944 vec
<constructor_elt
, va_gc
> *v
;
946 tree inner_type
= TREE_TYPE (type
);
947 tree width
= vector_element_bits_tree (type
);
948 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
949 tree comp_inner_type
= cond_type
;
950 tree index
= bitsize_int (0);
951 tree comp_width
= width
;
952 tree comp_index
= index
;
954 location_t loc
= gimple_location (gsi_stmt (*gsi
));
956 if (!is_gimple_val (a
))
958 gcc_assert (COMPARISON_CLASS_P (a
));
959 a_is_comparison
= true;
960 a1
= TREE_OPERAND (a
, 0);
961 a2
= TREE_OPERAND (a
, 1);
962 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
963 comp_width
= vector_element_bits_tree (TREE_TYPE (a1
));
966 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
), TREE_CODE (a
)))
969 /* Handle vector boolean types with bitmasks. If there is a comparison
970 and we can expand the comparison into the vector boolean bitmask,
971 or otherwise if it is compatible with type, we can transform
972 vbfld_1 = x_2 < y_3 ? vbfld_4 : vbfld_5;
975 tmp_7 = tmp_6 & vbfld_4;
977 tmp_9 = tmp_8 & vbfld_5;
978 vbfld_1 = tmp_7 | tmp_9;
979 Similarly for vbfld_10 instead of x_2 < y_3. */
980 if (VECTOR_BOOLEAN_TYPE_P (type
)
981 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
982 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
983 TYPE_VECTOR_SUBPARTS (type
)
984 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (type
))))
986 ? useless_type_conversion_p (type
, TREE_TYPE (a
))
987 : expand_vec_cmp_expr_p (TREE_TYPE (a1
), type
, TREE_CODE (a
))))
990 a
= gimplify_build2 (gsi
, TREE_CODE (a
), type
, a1
, a2
);
991 a1
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a
, b
);
992 a2
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, type
, a
);
993 a2
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a2
, c
);
994 a
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, type
, a1
, a2
);
995 gimple_assign_set_rhs_from_tree (gsi
, a
);
996 update_stmt (gsi_stmt (*gsi
));
1000 /* TODO: try and find a smaller vector type. */
1002 warning_at (loc
, OPT_Wvector_operation_performance
,
1003 "vector condition will be expanded piecewise");
1005 if (!a_is_comparison
1006 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1007 && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a
)))
1008 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a
))),
1009 TYPE_VECTOR_SUBPARTS (TREE_TYPE (a
))
1010 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1011 (TREE_TYPE (TREE_TYPE (a
))))))
1013 a_is_scalar_bitmask
= true;
1014 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a
)));
1015 tree atype
= build_nonstandard_integer_type (prec
, 1);
1016 a
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, atype
, a
);
1019 int nunits
= nunits_for_known_piecewise_op (type
);
1020 vec_alloc (v
, nunits
);
1021 for (i
= 0; i
< nunits
; i
++)
1024 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
1025 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
1026 if (a_is_comparison
)
1028 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
,
1029 comp_width
, comp_index
);
1030 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
,
1031 comp_width
, comp_index
);
1032 aa
= fold_build2 (TREE_CODE (a
), cond_type
, aa1
, aa2
);
1034 else if (a_is_scalar_bitmask
)
1036 wide_int w
= wi::set_bit_in_zero (i
, TYPE_PRECISION (TREE_TYPE (a
)));
1037 result
= gimplify_build2 (gsi
, BIT_AND_EXPR
, TREE_TYPE (a
),
1038 a
, wide_int_to_tree (TREE_TYPE (a
), w
));
1039 aa
= fold_build2 (NE_EXPR
, boolean_type_node
, result
,
1040 build_zero_cst (TREE_TYPE (a
)));
1043 aa
= tree_vec_extract (gsi
, cond_type
, a
, width
, index
);
1044 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
1045 constructor_elt ce
= {NULL_TREE
, result
};
1047 index
= int_const_binop (PLUS_EXPR
, index
, width
);
1048 if (width
== comp_width
)
1051 comp_index
= int_const_binop (PLUS_EXPR
, comp_index
, comp_width
);
1054 constr
= build_constructor (type
, v
);
1055 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1056 update_stmt (gsi_stmt (*gsi
));
1060 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
1061 gassign
*assign
, enum tree_code code
)
1063 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1065 /* If the compute mode is not a vector mode (hence we are not decomposing
1066 a BLKmode vector to smaller, hardware-supported vectors), we may want
1067 to expand the operations in parallel. */
1068 if (!VECTOR_MODE_P (compute_mode
))
1073 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1074 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
1075 gimple_assign_rhs1 (assign
),
1076 gimple_assign_rhs2 (assign
), code
);
1080 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1081 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
1082 gimple_assign_rhs1 (assign
),
1089 return expand_vector_parallel (gsi
, do_binop
, type
,
1090 gimple_assign_rhs1 (assign
),
1091 gimple_assign_rhs2 (assign
), code
);
1094 return expand_vector_parallel (gsi
, do_unop
, type
,
1095 gimple_assign_rhs1 (assign
),
1110 case UNORDERED_EXPR
:
1112 tree rhs1
= gimple_assign_rhs1 (assign
);
1113 tree rhs2
= gimple_assign_rhs2 (assign
);
1115 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
);
1118 case TRUNC_DIV_EXPR
:
1119 case TRUNC_MOD_EXPR
:
1121 tree rhs1
= gimple_assign_rhs1 (assign
);
1122 tree rhs2
= gimple_assign_rhs2 (assign
);
1126 || !VECTOR_INTEGER_TYPE_P (type
)
1127 || TREE_CODE (rhs2
) != VECTOR_CST
1128 || !VECTOR_MODE_P (TYPE_MODE (type
)))
1131 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
1132 if (ret
!= NULL_TREE
)
1141 if (TREE_CODE_CLASS (code
) == tcc_unary
)
1142 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
1143 gimple_assign_rhs1 (assign
),
1146 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
1147 gimple_assign_rhs1 (assign
),
1148 gimple_assign_rhs2 (assign
), code
);
1152 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1154 _9 = { b_7, b_7, b_7, b_7 };
1155 a_5 = _9 + { 0, 3, 6, 9 };
1156 because vector splat operation is usually more efficient
1157 than piecewise initialization of the vector. */
1160 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1162 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1163 tree lhs
= gimple_assign_lhs (stmt
);
1164 tree rhs
= gimple_assign_rhs1 (stmt
);
1165 tree type
= TREE_TYPE (rhs
);
1167 unsigned HOST_WIDE_INT nelts
;
1168 bool all_same
= true;
1169 constructor_elt
*elt
;
1171 tree base
= NULL_TREE
;
1174 if (!TYPE_VECTOR_SUBPARTS (type
).is_constant (&nelts
)
1176 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1178 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1179 if (op
== unknown_optab
1180 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1182 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1183 if (TREE_CODE (elt
->value
) != SSA_NAME
1184 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1188 tree this_base
= elt
->value
;
1189 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1191 for (j
= 0; j
< nelts
+ 1; j
++)
1193 g
= SSA_NAME_DEF_STMT (this_base
);
1194 if (is_gimple_assign (g
)
1195 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1196 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1197 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1198 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1199 this_base
= gimple_assign_rhs1 (g
);
1205 else if (this_base
!= base
)
1210 tree_vector_builder
cst (type
, nelts
, 1);
1211 for (i
= 0; i
< nelts
; i
++)
1213 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;
1214 tree elt
= build_zero_cst (TREE_TYPE (base
));
1215 while (this_base
!= base
)
1217 g
= SSA_NAME_DEF_STMT (this_base
);
1218 elt
= fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1219 elt
, gimple_assign_rhs2 (g
));
1220 if (elt
== NULL_TREE
1221 || TREE_CODE (elt
) != INTEGER_CST
1222 || TREE_OVERFLOW (elt
))
1224 this_base
= gimple_assign_rhs1 (g
);
1226 cst
.quick_push (elt
);
1228 for (i
= 0; i
< nelts
; i
++)
1229 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1230 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1231 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1232 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1234 gsi_replace (gsi
, g
, false);
1237 /* Return a type for the widest vector mode whose components are of type
1238 TYPE, or NULL_TREE if none is found. */
1241 type_for_widest_vector_mode (tree type
, optab op
)
1243 machine_mode inner_mode
= TYPE_MODE (type
);
1244 machine_mode best_mode
= VOIDmode
, mode
;
1245 poly_int64 best_nunits
= 0;
1247 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1248 mode
= MIN_MODE_VECTOR_FLOAT
;
1249 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1250 mode
= MIN_MODE_VECTOR_FRACT
;
1251 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1252 mode
= MIN_MODE_VECTOR_UFRACT
;
1253 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1254 mode
= MIN_MODE_VECTOR_ACCUM
;
1255 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1256 mode
= MIN_MODE_VECTOR_UACCUM
;
1257 else if (inner_mode
== BImode
)
1258 mode
= MIN_MODE_VECTOR_BOOL
;
1260 mode
= MIN_MODE_VECTOR_INT
;
1262 FOR_EACH_MODE_FROM (mode
, mode
)
1263 if (GET_MODE_INNER (mode
) == inner_mode
1264 && maybe_gt (GET_MODE_NUNITS (mode
), best_nunits
)
1265 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
1266 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1268 if (best_mode
== VOIDmode
)
1271 return build_vector_type_for_mode (type
, best_mode
);
1275 /* Build a reference to the element of the vector VECT. Function
1276 returns either the element itself, either BIT_FIELD_REF, or an
1277 ARRAY_REF expression.
1279 GSI is required to insert temporary variables while building a
1280 refernece to the element of the vector VECT.
1282 PTMPVEC is a pointer to the temporary variable for caching
1283 purposes. In case when PTMPVEC is NULL new temporary variable
1286 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1288 tree vect_type
, vect_elt_type
;
1292 bool need_asgn
= true;
1293 unsigned int elements
;
1295 vect_type
= TREE_TYPE (vect
);
1296 vect_elt_type
= TREE_TYPE (vect_type
);
1297 elements
= nunits_for_known_piecewise_op (vect_type
);
1299 if (TREE_CODE (idx
) == INTEGER_CST
)
1301 unsigned HOST_WIDE_INT index
;
1303 /* Given that we're about to compute a binary modulus,
1304 we don't care about the high bits of the value. */
1305 index
= TREE_INT_CST_LOW (idx
);
1306 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1308 index
&= elements
- 1;
1309 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1312 /* When lowering a vector statement sequence do some easy
1313 simplification by looking through intermediate vector results. */
1314 if (TREE_CODE (vect
) == SSA_NAME
)
1316 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1317 if (is_gimple_assign (def_stmt
)
1318 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1319 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1320 vect
= gimple_assign_rhs1 (def_stmt
);
1323 if (TREE_CODE (vect
) == VECTOR_CST
)
1324 return VECTOR_CST_ELT (vect
, index
);
1325 else if (TREE_CODE (vect
) == CONSTRUCTOR
1326 && (CONSTRUCTOR_NELTS (vect
) == 0
1327 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1330 if (index
< CONSTRUCTOR_NELTS (vect
))
1331 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1332 return build_zero_cst (vect_elt_type
);
1336 tree size
= vector_element_bits_tree (vect_type
);
1337 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1339 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1344 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1346 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1355 TREE_ADDRESSABLE (tmpvec
) = 1;
1356 asgn
= gimple_build_assign (tmpvec
, vect
);
1357 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1360 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1361 return build4 (ARRAY_REF
, vect_elt_type
,
1362 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1363 idx
, NULL_TREE
, NULL_TREE
);
1366 /* Check if VEC_PERM_EXPR within the given setting is supported
1367 by hardware, or lower it piecewise.
1369 When VEC_PERM_EXPR has the same first and second operands:
1370 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1371 {v0[mask[0]], v0[mask[1]], ...}
1372 MASK and V0 must have the same number of elements.
1374 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1375 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1376 V0 and V1 must have the same type. MASK, V0, V1 must have the
1377 same number of arguments. */
1380 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1382 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1383 tree mask
= gimple_assign_rhs3 (stmt
);
1384 tree vec0
= gimple_assign_rhs1 (stmt
);
1385 tree vec1
= gimple_assign_rhs2 (stmt
);
1386 tree vect_type
= TREE_TYPE (vec0
);
1387 tree mask_type
= TREE_TYPE (mask
);
1388 tree vect_elt_type
= TREE_TYPE (vect_type
);
1389 tree mask_elt_type
= TREE_TYPE (mask_type
);
1390 unsigned HOST_WIDE_INT elements
;
1391 vec
<constructor_elt
, va_gc
> *v
;
1392 tree constr
, t
, si
, i_val
;
1393 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1394 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1395 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1398 if (!TYPE_VECTOR_SUBPARTS (vect_type
).is_constant (&elements
))
1401 if (TREE_CODE (mask
) == SSA_NAME
)
1403 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1404 if (is_gimple_assign (def_stmt
)
1405 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1406 mask
= gimple_assign_rhs1 (def_stmt
);
1409 vec_perm_builder sel_int
;
1411 if (TREE_CODE (mask
) == VECTOR_CST
1412 && tree_to_vec_perm_builder (&sel_int
, mask
))
1414 vec_perm_indices
indices (sel_int
, 2, elements
);
1415 if (can_vec_perm_const_p (TYPE_MODE (vect_type
), indices
))
1417 gimple_assign_set_rhs3 (stmt
, mask
);
1421 /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1422 vector as VEC1 and a right element shift MASK. */
1423 if (optab_handler (vec_shr_optab
, TYPE_MODE (vect_type
))
1425 && TREE_CODE (vec1
) == VECTOR_CST
1426 && initializer_zerop (vec1
)
1427 && maybe_ne (indices
[0], 0)
1428 && known_lt (poly_uint64 (indices
[0]), elements
))
1430 bool ok_p
= indices
.series_p (0, 1, indices
[0], 1);
1433 for (i
= 1; i
< elements
; ++i
)
1435 poly_uint64 actual
= indices
[i
];
1436 poly_uint64 expected
= i
+ indices
[0];
1437 /* Indices into the second vector are all equivalent. */
1438 if (maybe_lt (actual
, elements
)
1439 ? maybe_ne (actual
, expected
)
1440 : maybe_lt (expected
, elements
))
1443 ok_p
= i
== elements
;
1447 gimple_assign_set_rhs3 (stmt
, mask
);
1452 /* And similarly vec_shl pattern. */
1453 if (optab_handler (vec_shl_optab
, TYPE_MODE (vect_type
))
1455 && TREE_CODE (vec0
) == VECTOR_CST
1456 && initializer_zerop (vec0
))
1458 unsigned int first
= 0;
1459 for (i
= 0; i
< elements
; ++i
)
1460 if (known_eq (poly_uint64 (indices
[i
]), elements
))
1462 if (i
== 0 || first
)
1467 ? maybe_ne (poly_uint64 (indices
[i
]),
1468 elements
+ i
- first
)
1469 : maybe_ge (poly_uint64 (indices
[i
]), elements
))
1473 gimple_assign_set_rhs3 (stmt
, mask
);
1479 else if (can_vec_perm_var_p (TYPE_MODE (vect_type
)))
1482 warning_at (loc
, OPT_Wvector_operation_performance
,
1483 "vector shuffling operation will be expanded piecewise");
1485 vec_alloc (v
, elements
);
1486 for (i
= 0; i
< elements
; i
++)
1489 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1491 if (TREE_CODE (i_val
) == INTEGER_CST
)
1493 unsigned HOST_WIDE_INT index
;
1495 index
= TREE_INT_CST_LOW (i_val
);
1496 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1497 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1499 if (two_operand_p
&& (index
& elements
) != 0)
1500 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1502 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1504 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1505 true, GSI_SAME_STMT
);
1509 tree cond
= NULL_TREE
, v0_val
;
1513 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1514 build_int_cst (mask_elt_type
, elements
));
1515 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1516 true, GSI_SAME_STMT
);
1519 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1520 build_int_cst (mask_elt_type
, elements
- 1));
1521 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1522 true, GSI_SAME_STMT
);
1524 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1525 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1526 true, GSI_SAME_STMT
);
1532 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1533 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1534 true, GSI_SAME_STMT
);
1536 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1537 cond
, build_zero_cst (mask_elt_type
));
1538 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1539 cond
, v0_val
, v1_val
);
1540 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1541 true, GSI_SAME_STMT
);
1547 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1550 constr
= build_constructor (vect_type
, v
);
1551 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1552 update_stmt (gsi_stmt (*gsi
));
1555 /* If OP is a uniform vector return the element it is a splat from. */
1558 ssa_uniform_vector_p (tree op
)
1560 if (TREE_CODE (op
) == VECTOR_CST
1561 || TREE_CODE (op
) == VEC_DUPLICATE_EXPR
1562 || TREE_CODE (op
) == CONSTRUCTOR
)
1563 return uniform_vector_p (op
);
1564 if (TREE_CODE (op
) == SSA_NAME
)
1566 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1567 if (gimple_assign_single_p (def_stmt
))
1568 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1573 /* Return type in which CODE operation with optab OP can be
1577 get_compute_type (enum tree_code code
, optab op
, tree type
)
1579 /* For very wide vectors, try using a smaller vector mode. */
1580 tree compute_type
= type
;
1582 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1583 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1585 tree vector_compute_type
1586 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1587 if (vector_compute_type
!= NULL_TREE
1588 && subparts_gt (compute_type
, vector_compute_type
)
1589 && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type
), 1U)
1590 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1591 != CODE_FOR_nothing
))
1592 compute_type
= vector_compute_type
;
1595 /* If we are breaking a BLKmode vector into smaller pieces,
1596 type_for_widest_vector_mode has already looked into the optab,
1597 so skip these checks. */
1598 if (compute_type
== type
)
1600 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1601 if (VECTOR_MODE_P (compute_mode
))
1603 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1604 return compute_type
;
1605 if (code
== MULT_HIGHPART_EXPR
1606 && can_mult_highpart_p (compute_mode
,
1607 TYPE_UNSIGNED (compute_type
)))
1608 return compute_type
;
1610 /* There is no operation in hardware, so fall back to scalars. */
1611 compute_type
= TREE_TYPE (type
);
1614 return compute_type
;
1618 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1619 tree bitpos
, tree bitsize
, enum tree_code code
,
1620 tree type ATTRIBUTE_UNUSED
)
1622 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
1623 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1624 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
1625 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1626 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1627 return gimplify_build3 (gsi
, code
, inner_type
, unshare_expr (cond
), a
, b
);
1630 /* Expand a vector COND_EXPR to scalars, piecewise. */
1632 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1634 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1635 tree type
= gimple_expr_type (stmt
);
1636 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1637 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1638 gcc_assert (compute_mode
!= BLKmode
);
1639 tree lhs
= gimple_assign_lhs (stmt
);
1640 tree rhs2
= gimple_assign_rhs2 (stmt
);
1641 tree rhs3
= gimple_assign_rhs3 (stmt
);
1644 /* If the compute mode is not a vector mode (hence we are not decomposing
1645 a BLKmode vector to smaller, hardware-supported vectors), we may want
1646 to expand the operations in parallel. */
1647 if (!VECTOR_MODE_P (compute_mode
))
1648 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1651 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1652 rhs2
, rhs3
, COND_EXPR
);
1653 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1654 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1657 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1658 way to do it is change expand_vector_operation and its callees to
1659 return a tree_code, RHS1 and RHS2 instead of a tree. */
1660 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1661 update_stmt (gsi_stmt (*gsi
));
1664 /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1665 lowering. If INNER_TYPE is not a vector type, this is a scalar
1669 do_vec_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1670 tree decl
, tree bitpos
, tree bitsize
,
1671 enum tree_code code
, tree type
)
1673 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1674 if (!VECTOR_TYPE_P (inner_type
))
1675 return gimplify_build1 (gsi
, code
, TREE_TYPE (type
), a
);
1676 if (code
== CALL_EXPR
)
1678 gimple
*g
= gimple_build_call (decl
, 1, a
);
1679 tree lhs
= make_ssa_name (TREE_TYPE (TREE_TYPE (decl
)));
1680 gimple_call_set_lhs (g
, lhs
);
1681 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1686 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1687 TYPE_VECTOR_SUBPARTS (inner_type
));
1688 return gimplify_build1 (gsi
, code
, outer_type
, a
);
1692 /* Similarly, but for narrowing conversion. */
1695 do_vec_narrow_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1696 tree
, tree bitpos
, tree
, enum tree_code code
,
1699 tree itype
= build_vector_type (TREE_TYPE (inner_type
),
1700 exact_div (TYPE_VECTOR_SUBPARTS (inner_type
),
1702 tree b
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
), bitpos
);
1703 tree c
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
),
1704 int_const_binop (PLUS_EXPR
, bitpos
,
1705 TYPE_SIZE (itype
)));
1706 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1707 TYPE_VECTOR_SUBPARTS (inner_type
));
1708 return gimplify_build2 (gsi
, code
, outer_type
, b
, c
);
1711 /* Expand VEC_CONVERT ifn call. */
1714 expand_vector_conversion (gimple_stmt_iterator
*gsi
)
1716 gimple
*stmt
= gsi_stmt (*gsi
);
1718 tree lhs
= gimple_call_lhs (stmt
);
1719 tree arg
= gimple_call_arg (stmt
, 0);
1720 tree ret_type
= TREE_TYPE (lhs
);
1721 tree arg_type
= TREE_TYPE (arg
);
1722 tree new_rhs
, compute_type
= TREE_TYPE (arg_type
);
1723 enum tree_code code
= NOP_EXPR
;
1724 enum tree_code code1
= ERROR_MARK
;
1725 enum { NARROW
, NONE
, WIDEN
} modifier
= NONE
;
1726 optab optab1
= unknown_optab
;
1728 gcc_checking_assert (VECTOR_TYPE_P (ret_type
) && VECTOR_TYPE_P (arg_type
));
1729 if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type
))
1730 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type
)))
1731 code
= FIX_TRUNC_EXPR
;
1732 else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type
))
1733 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type
)))
1735 unsigned int ret_elt_bits
= vector_element_bits (ret_type
);
1736 unsigned int arg_elt_bits
= vector_element_bits (arg_type
);
1737 if (ret_elt_bits
< arg_elt_bits
)
1739 else if (ret_elt_bits
> arg_elt_bits
)
1742 if (modifier
== NONE
&& (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
))
1744 if (supportable_convert_operation (code
, ret_type
, arg_type
, &code1
))
1746 g
= gimple_build_assign (lhs
, code1
, arg
);
1747 gsi_replace (gsi
, g
, false);
1750 /* Can't use get_compute_type here, as supportable_convert_operation
1751 doesn't necessarily use an optab and needs two arguments. */
1752 tree vec_compute_type
1753 = type_for_widest_vector_mode (TREE_TYPE (arg_type
), mov_optab
);
1754 if (vec_compute_type
1755 && VECTOR_MODE_P (TYPE_MODE (vec_compute_type
))
1756 && subparts_gt (arg_type
, vec_compute_type
))
1758 unsigned HOST_WIDE_INT nelts
1759 = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type
));
1762 tree ret1_type
= build_vector_type (TREE_TYPE (ret_type
), nelts
);
1763 tree arg1_type
= build_vector_type (TREE_TYPE (arg_type
), nelts
);
1764 if (supportable_convert_operation (code
, ret1_type
, arg1_type
,
1767 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
,
1768 ret_type
, arg1_type
, arg
,
1770 g
= gimple_build_assign (lhs
, new_rhs
);
1771 gsi_replace (gsi
, g
, false);
1778 else if (modifier
== NARROW
)
1783 code1
= VEC_PACK_TRUNC_EXPR
;
1784 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1786 case FIX_TRUNC_EXPR
:
1787 code1
= VEC_PACK_FIX_TRUNC_EXPR
;
1788 /* The signedness is determined from output operand. */
1789 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1792 code1
= VEC_PACK_FLOAT_EXPR
;
1793 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1800 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1801 enum insn_code icode1
;
1802 if (VECTOR_TYPE_P (compute_type
)
1803 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1804 != CODE_FOR_nothing
)
1805 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
))
1808 = build_vector_type (TREE_TYPE (ret_type
),
1809 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1810 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1812 if (compute_type
== arg_type
)
1814 new_rhs
= gimplify_build2 (gsi
, code1
, cretd_type
,
1815 arg
, build_zero_cst (arg_type
));
1816 new_rhs
= tree_vec_extract (gsi
, ret_type
, new_rhs
,
1817 TYPE_SIZE (ret_type
),
1819 g
= gimple_build_assign (lhs
, new_rhs
);
1820 gsi_replace (gsi
, g
, false);
1824 = build_vector_type (TREE_TYPE (compute_type
),
1825 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1826 if (TYPE_MAIN_VARIANT (dcompute_type
)
1827 == TYPE_MAIN_VARIANT (arg_type
))
1828 new_rhs
= do_vec_narrow_conversion (gsi
, dcompute_type
, arg
,
1829 NULL_TREE
, bitsize_int (0),
1833 new_rhs
= expand_vector_piecewise (gsi
,
1834 do_vec_narrow_conversion
,
1835 arg_type
, dcompute_type
,
1836 arg
, NULL_TREE
, code1
,
1838 g
= gimple_build_assign (lhs
, new_rhs
);
1839 gsi_replace (gsi
, g
, false);
1844 else if (modifier
== WIDEN
)
1846 enum tree_code code2
= ERROR_MARK
;
1847 optab optab2
= unknown_optab
;
1851 code1
= VEC_UNPACK_LO_EXPR
;
1852 code2
= VEC_UNPACK_HI_EXPR
;
1854 case FIX_TRUNC_EXPR
:
1855 code1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
1856 code2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
1859 code1
= VEC_UNPACK_FLOAT_LO_EXPR
;
1860 code2
= VEC_UNPACK_FLOAT_HI_EXPR
;
1865 if (BYTES_BIG_ENDIAN
)
1866 std::swap (code1
, code2
);
1868 if (code
== FIX_TRUNC_EXPR
)
1870 /* The signedness is determined from output operand. */
1871 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1872 optab2
= optab_for_tree_code (code2
, ret_type
, optab_default
);
1876 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1877 optab2
= optab_for_tree_code (code2
, arg_type
, optab_default
);
1880 if (optab1
&& optab2
)
1881 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1883 enum insn_code icode1
, icode2
;
1884 if (VECTOR_TYPE_P (compute_type
)
1885 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1886 != CODE_FOR_nothing
)
1887 && ((icode2
= optab_handler (optab2
, TYPE_MODE (compute_type
)))
1888 != CODE_FOR_nothing
)
1889 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
)
1890 && (insn_data
[icode1
].operand
[0].mode
1891 == insn_data
[icode2
].operand
[0].mode
))
1894 = exact_div (TYPE_VECTOR_SUBPARTS (compute_type
), 2);
1895 tree cretd_type
= build_vector_type (TREE_TYPE (ret_type
), nunits
);
1896 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1898 vec
<constructor_elt
, va_gc
> *v
;
1899 tree part_width
= TYPE_SIZE (compute_type
);
1900 tree index
= bitsize_int (0);
1901 int nunits
= nunits_for_known_piecewise_op (arg_type
);
1902 int delta
= tree_to_uhwi (part_width
) / arg_elt_bits
;
1904 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1906 if (compute_type
!= arg_type
)
1907 warning_at (loc
, OPT_Wvector_operation_performance
,
1908 "vector operation will be expanded piecewise");
1915 vec_alloc (v
, (nunits
+ delta
- 1) / delta
* 2);
1916 for (i
= 0; i
< nunits
;
1917 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
,
1921 if (compute_type
!= arg_type
)
1922 a
= tree_vec_extract (gsi
, compute_type
, a
, part_width
,
1924 tree result
= gimplify_build1 (gsi
, code1
, cretd_type
, a
);
1925 constructor_elt ce
= { NULL_TREE
, result
};
1927 ce
.value
= gimplify_build1 (gsi
, code2
, cretd_type
, a
);
1931 new_rhs
= build_constructor (ret_type
, v
);
1932 g
= gimple_build_assign (lhs
, new_rhs
);
1933 gsi_replace (gsi
, g
, false);
1939 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
, arg_type
,
1940 TREE_TYPE (arg_type
), arg
,
1941 NULL_TREE
, code
, ret_type
);
1942 g
= gimple_build_assign (lhs
, new_rhs
);
1943 gsi_replace (gsi
, g
, false);
1946 /* Process one statement. If we identify a vector operation, expand it. */
1949 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
)
1951 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
1952 enum tree_code code
;
1953 optab op
= unknown_optab
;
1954 enum gimple_rhs_class rhs_class
;
1957 /* Only consider code == GIMPLE_ASSIGN. */
1958 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
1961 if (gimple_call_internal_p (gsi_stmt (*gsi
), IFN_VEC_CONVERT
))
1962 expand_vector_conversion (gsi
);
1966 code
= gimple_assign_rhs_code (stmt
);
1967 rhs_class
= get_gimple_rhs_class (code
);
1968 lhs
= gimple_assign_lhs (stmt
);
1970 if (code
== VEC_PERM_EXPR
)
1972 lower_vec_perm (gsi
);
1976 if (code
== VEC_COND_EXPR
)
1978 expand_vector_condition (gsi
);
1982 if (code
== COND_EXPR
1983 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
1984 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
1986 expand_vector_scalar_condition (gsi
);
1990 if (code
== CONSTRUCTOR
1991 && TREE_CODE (lhs
) == SSA_NAME
1992 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
1993 && !gimple_clobber_p (stmt
)
1996 optimize_vector_constructor (gsi
);
2000 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
2003 rhs1
= gimple_assign_rhs1 (stmt
);
2004 type
= gimple_expr_type (stmt
);
2005 if (rhs_class
== GIMPLE_BINARY_RHS
)
2006 rhs2
= gimple_assign_rhs2 (stmt
);
2008 if (!VECTOR_TYPE_P (type
)
2009 || !VECTOR_TYPE_P (TREE_TYPE (rhs1
)))
2012 /* A scalar operation pretending to be a vector one. */
2013 if (VECTOR_BOOLEAN_TYPE_P (type
)
2014 && !VECTOR_MODE_P (TYPE_MODE (type
))
2015 && TYPE_MODE (type
) != BLKmode
2016 && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) != tcc_comparison
2017 || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
))
2018 && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1
)))
2019 && TYPE_MODE (TREE_TYPE (rhs1
)) != BLKmode
)))
2022 /* If the vector operation is operating on all same vector elements
2023 implement it with a scalar operation and a splat if the target
2024 supports the scalar operation. */
2025 tree srhs1
, srhs2
= NULL_TREE
;
2026 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
2027 && (rhs2
== NULL_TREE
2028 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
2030 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2031 /* As we query direct optabs restrict to non-convert operations. */
2032 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
2034 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
2035 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
2036 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
2038 tree slhs
= make_ssa_name (TREE_TYPE (srhs1
));
2039 gimple
*repl
= gimple_build_assign (slhs
, code
, srhs1
, srhs2
);
2040 gsi_insert_before (gsi
, repl
, GSI_SAME_STMT
);
2041 gimple_assign_set_rhs_from_tree (gsi
,
2042 build_vector_from_val (type
, slhs
));
2048 if (CONVERT_EXPR_CODE_P (code
)
2049 || code
== FLOAT_EXPR
2050 || code
== FIX_TRUNC_EXPR
2051 || code
== VIEW_CONVERT_EXPR
)
2054 /* The signedness is determined from input argument. */
2055 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
2056 || code
== VEC_UNPACK_FLOAT_LO_EXPR
2057 || code
== VEC_PACK_FLOAT_EXPR
)
2059 /* We do not know how to scalarize those. */
2063 /* For widening/narrowing vector operations, the relevant type is of the
2064 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2065 calculated in the same way above. */
2066 if (code
== WIDEN_SUM_EXPR
2067 || code
== VEC_WIDEN_MULT_HI_EXPR
2068 || code
== VEC_WIDEN_MULT_LO_EXPR
2069 || code
== VEC_WIDEN_MULT_EVEN_EXPR
2070 || code
== VEC_WIDEN_MULT_ODD_EXPR
2071 || code
== VEC_UNPACK_HI_EXPR
2072 || code
== VEC_UNPACK_LO_EXPR
2073 || code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
2074 || code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
2075 || code
== VEC_PACK_TRUNC_EXPR
2076 || code
== VEC_PACK_SAT_EXPR
2077 || code
== VEC_PACK_FIX_TRUNC_EXPR
2078 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
2079 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
2081 /* We do not know how to scalarize those. */
2085 /* Choose between vector shift/rotate by vector and vector shift/rotate by
2087 if (code
== LSHIFT_EXPR
2088 || code
== RSHIFT_EXPR
2089 || code
== LROTATE_EXPR
2090 || code
== RROTATE_EXPR
)
2094 /* Check whether we have vector <op> {x,x,x,x} where x
2095 could be a scalar variable or a constant. Transform
2096 vector <op> {x,x,x,x} ==> vector <op> scalar. */
2097 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2101 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2103 gimple_assign_set_rhs2 (stmt
, first
);
2109 opv
= optab_for_tree_code (code
, type
, optab_vector
);
2110 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2114 op
= optab_for_tree_code (code
, type
, optab_scalar
);
2116 compute_type
= get_compute_type (code
, op
, type
);
2117 if (compute_type
== type
)
2119 /* The rtl expander will expand vector/scalar as vector/vector
2120 if necessary. Pick one with wider vector type. */
2121 tree compute_vtype
= get_compute_type (code
, opv
, type
);
2122 if (subparts_gt (compute_vtype
, compute_type
))
2124 compute_type
= compute_vtype
;
2129 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
2131 if (compute_type
== NULL_TREE
)
2132 compute_type
= get_compute_type (code
, op
, type
);
2133 if (compute_type
== type
)
2135 /* Before splitting vector rotates into scalar rotates,
2136 see if we can't use vector shifts and BIT_IOR_EXPR
2137 instead. For vector by vector rotates we'd also
2138 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2139 for now, fold doesn't seem to create such rotates anyway. */
2140 if (compute_type
== TREE_TYPE (type
)
2141 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2143 optab oplv
= vashl_optab
, opl
= ashl_optab
;
2144 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
2145 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
2146 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
2147 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
2148 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
2149 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
2150 /* The rtl expander will expand vector/scalar as vector/vector
2151 if necessary. Pick one with wider vector type. */
2152 if (subparts_gt (compute_lvtype
, compute_ltype
))
2154 compute_ltype
= compute_lvtype
;
2157 if (subparts_gt (compute_rvtype
, compute_rtype
))
2159 compute_rtype
= compute_rvtype
;
2162 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2164 compute_type
= compute_ltype
;
2165 if (subparts_gt (compute_type
, compute_rtype
))
2166 compute_type
= compute_rtype
;
2167 if (subparts_gt (compute_type
, compute_otype
))
2168 compute_type
= compute_otype
;
2169 /* Verify all 3 operations can be performed in that type. */
2170 if (compute_type
!= TREE_TYPE (type
))
2172 if (optab_handler (opl
, TYPE_MODE (compute_type
))
2174 || optab_handler (opr
, TYPE_MODE (compute_type
))
2176 || optab_handler (opo
, TYPE_MODE (compute_type
))
2177 == CODE_FOR_nothing
)
2178 compute_type
= TREE_TYPE (type
);
2184 op
= optab_for_tree_code (code
, type
, optab_default
);
2186 /* Optabs will try converting a negation into a subtraction, so
2187 look for it as well. TODO: negation of floating-point vectors
2188 might be turned into an exclusive OR toggling the sign bit. */
2189 if (op
== unknown_optab
2190 && code
== NEGATE_EXPR
2191 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
2192 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
2194 if (compute_type
== NULL_TREE
)
2195 compute_type
= get_compute_type (code
, op
, type
);
2196 if (compute_type
== type
)
2199 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
);
2201 /* Leave expression untouched for later expansion. */
2202 if (new_rhs
== NULL_TREE
)
2205 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
2206 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
2209 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2210 way to do it is change expand_vector_operation and its callees to
2211 return a tree_code, RHS1 and RHS2 instead of a tree. */
2212 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
2213 update_stmt (gsi_stmt (*gsi
));
2216 /* Use this to lower vector operations introduced by the vectorizer,
2217 if it may need the bit-twiddling tricks implemented in this file. */
2220 expand_vector_operations (void)
2222 gimple_stmt_iterator gsi
;
2224 bool cfg_changed
= false;
2226 FOR_EACH_BB_FN (bb
, cfun
)
2228 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2230 expand_vector_operations_1 (&gsi
);
2231 /* ??? If we do not cleanup EH then we will ICE in
2232 verification. But in reality we have created wrong-code
2233 as we did not properly transition EH info and edges to
2234 the piecewise computations. */
2235 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
2236 && gimple_purge_dead_eh_edges (bb
))
2241 return cfg_changed
? TODO_cleanup_cfg
: 0;
2246 const pass_data pass_data_lower_vector
=
2248 GIMPLE_PASS
, /* type */
2249 "veclower", /* name */
2250 OPTGROUP_VEC
, /* optinfo_flags */
2251 TV_NONE
, /* tv_id */
2252 PROP_cfg
, /* properties_required */
2253 PROP_gimple_lvec
, /* properties_provided */
2254 0, /* properties_destroyed */
2255 0, /* todo_flags_start */
2256 TODO_update_ssa
, /* todo_flags_finish */
2259 class pass_lower_vector
: public gimple_opt_pass
2262 pass_lower_vector (gcc::context
*ctxt
)
2263 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
2266 /* opt_pass methods: */
2267 virtual bool gate (function
*fun
)
2269 return !(fun
->curr_properties
& PROP_gimple_lvec
);
2272 virtual unsigned int execute (function
*)
2274 return expand_vector_operations ();
2277 }; // class pass_lower_vector
2282 make_pass_lower_vector (gcc::context
*ctxt
)
2284 return new pass_lower_vector (ctxt
);
2289 const pass_data pass_data_lower_vector_ssa
=
2291 GIMPLE_PASS
, /* type */
2292 "veclower2", /* name */
2293 OPTGROUP_VEC
, /* optinfo_flags */
2294 TV_NONE
, /* tv_id */
2295 PROP_cfg
, /* properties_required */
2296 PROP_gimple_lvec
, /* properties_provided */
2297 0, /* properties_destroyed */
2298 0, /* todo_flags_start */
2300 | TODO_cleanup_cfg
), /* todo_flags_finish */
2303 class pass_lower_vector_ssa
: public gimple_opt_pass
2306 pass_lower_vector_ssa (gcc::context
*ctxt
)
2307 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
2310 /* opt_pass methods: */
2311 opt_pass
* clone () { return new pass_lower_vector_ssa (m_ctxt
); }
2312 virtual unsigned int execute (function
*)
2314 return expand_vector_operations ();
2317 }; // class pass_lower_vector_ssa
2322 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
2324 return new pass_lower_vector_ssa (ctxt
);
2327 #include "gt-tree-vect-generic.h"