static bool vect_can_advance_ivs_p (loop_vec_info);
+/* Return the smallest scalar part of STMT.
+ This is used to determine the vectype of the stmt. We generally set the
+ vectype according to the type of the result (lhs). For stmts whose
+ result-type is different than the type of the arguments (e.g., demotion,
+ promotion), vectype will be reset appropriately (later). Note that we have
+ to visit the smallest datatype in this function, because that determines the
+ VF. If the smallest datatype in the loop is present only as the rhs of a
+ promotion operation - we'd miss it.
+ Such a case, where a variable of this datatype does not appear in the lhs
+ anywhere in the loop, can only occur if it's an invariant: e.g.:
+ 'int_x = (int) short_inv', which we'd expect to have been optimized away by
+ invariant motion. However, we cannot rely on invariant motion to always take
+ invariants out of the loop, and so in the case of promotion we also have to
+ check the rhs.
+ LHS_SIZE_UNIT and RHS_SIZE_UNIT contain the sizes of the corresponding
+ types. */
+
+tree
+vect_get_smallest_scalar_type (gimple stmt, HOST_WIDE_INT *lhs_size_unit,
+ HOST_WIDE_INT *rhs_size_unit)
+{
+ tree scalar_type = gimple_expr_type (stmt);
+ HOST_WIDE_INT lhs, rhs;
+
+ lhs = rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
+
+ if (is_gimple_assign (stmt)
+ && (gimple_assign_cast_p (stmt)
+ || gimple_assign_rhs_code (stmt) == WIDEN_MULT_EXPR
+ || gimple_assign_rhs_code (stmt) == FLOAT_EXPR))
+ {
+ tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
+
+ rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
+ if (rhs < lhs)
+ scalar_type = rhs_type;
+ }
+
+ *lhs_size_unit = lhs;
+ *rhs_size_unit = rhs;
+ return scalar_type;
+}
+
+
/* Function vect_determine_vectorization_factor
Determine the vectorization factor (VF). VF is the number of data elements
unsigned int nunits;
stmt_vec_info stmt_info;
int i;
+ HOST_WIDE_INT dummy;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
gcc_assert (! STMT_VINFO_DATA_REF (stmt_info)
&& !is_pattern_stmt_p (stmt_info));
- /* We generally set the vectype according to the type of the
- result (lhs).
- For stmts whose result-type is different than the type of the
- arguments (e.g. demotion, promotion), vectype will be reset
- appropriately (later). Note that we have to visit the smallest
- datatype in this function, because that determines the VF.
- If the smallest datatype in the loop is present only as the
- rhs of a promotion operation - we'd miss it here.
- Such a case, where a variable of this datatype does not appear
- in the lhs anywhere in the loop, can only occur if it's an
- invariant: e.g.: 'int_x = (int) short_inv', which we'd expect
- to have been optimized away by invariant motion. However, we
- cannot rely on invariant motion to always take invariants out
- of the loop, and so in the case of promotion we also have to
- check the rhs. */
- scalar_type = gimple_expr_type (stmt);
-
- if (is_gimple_assign (stmt)
- && (gimple_assign_cast_p (stmt)
- || gimple_assign_rhs_code (stmt) == WIDEN_MULT_EXPR
- || gimple_assign_rhs_code (stmt) == FLOAT_EXPR))
- {
- tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
- if (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type))
- < TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type)))
- scalar_type = rhs_type;
- }
-
+ scalar_type = vect_get_smallest_scalar_type (stmt, &dummy,
+ &dummy);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "get vectype for scalar type: ");
tree first_stmt_const_oprnd = NULL_TREE;
struct data_reference *first_dr;
bool pattern0 = false, pattern1 = false;
+ HOST_WIDE_INT dummy;
/* For every stmt in NODE find its def stmt/s. */
for (i = 0; VEC_iterate (gimple, stmts, i, stmt); i++)
return false;
}
- scalar_type = TREE_TYPE (lhs);
+ scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
vectype = get_vectype_for_scalar_type (scalar_type);
if (!vectype)
{
/* Load. */
if (i == 0)
{
- /* In case of multiple types we need to detect the smallest
- type. */
- if (*max_nunits < TYPE_VECTOR_SUBPARTS (vectype))
- *max_nunits = TYPE_VECTOR_SUBPARTS (vectype);
-
/* First stmt of the SLP group should be the first load of
the interleaving loop if data permutation is not allowed.
Check that there is no gap between the loads. */
/* For constant and loop invariant defs of SLP_NODE this function returns
(vector) defs (VEC_OPRNDS) that will be used in the vectorized stmts.
OP_NUM determines if we gather defs for operand 0 or operand 1 of the scalar
- stmts. */
+ stmts. NUMBER_OF_VECTORS is the number of vector defs to create. */
static void
vect_get_constant_vectors (slp_tree slp_node, VEC(tree,heap) **vec_oprnds,
- unsigned int op_num)
+ unsigned int op_num, unsigned int number_of_vectors)
{
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (slp_node);
gimple stmt = VEC_index (gimple, stmts, 0);
unsigned int vec_num, i;
int number_of_copies = 1;
bool is_store = false;
- unsigned int number_of_vectors = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
VEC (tree, heap) *voprnds = VEC_alloc (tree, heap, number_of_vectors);
bool constant_p;
gimple first_stmt;
enum tree_code code;
int number_of_vects;
+ HOST_WIDE_INT lhs_size_unit, rhs_size_unit;
+ first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
/* The number of vector defs is determined by the number of vector statements
in the node from which we get those statements. */
if (SLP_TREE_LEFT (slp_node))
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node));
else
- number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ {
+ number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
+ /* Number of vector stmts was calculated according to LHS in
+ vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
+ necessary. See vect_get_smallest_scalar_type() for details. */
+ vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
+ &rhs_size_unit);
+ if (rhs_size_unit != lhs_size_unit)
+ {
+ number_of_vects *= rhs_size_unit;
+ number_of_vects /= lhs_size_unit;
+ }
+ }
/* Allocate memory for vectorized defs. */
*vec_oprnds0 = VEC_alloc (tree, heap, number_of_vects);
vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node), vec_oprnds0);
else
/* Build vectors from scalar defs. */
- vect_get_constant_vectors (slp_node, vec_oprnds0, 0);
+ vect_get_constant_vectors (slp_node, vec_oprnds0, 0, number_of_vects);
- first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))
/* Since we don't call this function with loads, this is a group of
stores. */
vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node), vec_oprnds1);
else
/* Build vectors from scalar defs. */
- vect_get_constant_vectors (slp_node, vec_oprnds1, 1);
+ vect_get_constant_vectors (slp_node, vec_oprnds1, 1, number_of_vects);
}
projects / thirdparty / gcc.git / commitdiff
