/* Data References Analysis and Manipulation Utilities for Vectorization.
- Copyright (C) 2003-2018 Free Software Foundation, Inc.
+ Copyright (C) 2003-2020 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "tree-vectorizer.h"
#include "expr.h"
#include "builtins.h"
-#include "params.h"
#include "tree-cfg.h"
#include "tree-hash-traits.h"
#include "vec-perm-indices.h"
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "no array mode for %s["
- HOST_WIDE_INT_PRINT_DEC "]\n",
+ "no array mode for %s[%wu]\n",
GET_MODE_NAME (mode), count);
return false;
}
if (rhs < lhs)
scalar_type = rhs_type;
}
+ else if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
+ {
+ unsigned int i = 0;
+ if (gimple_call_internal_p (call))
+ {
+ internal_fn ifn = gimple_call_internal_fn (call);
+ if (internal_load_fn_p (ifn) || internal_store_fn_p (ifn))
+ /* gimple_expr_type already picked the type of the loaded
+ or stored data. */
+ i = ~0U;
+ else if (internal_fn_mask_index (ifn) == 0)
+ i = 1;
+ }
+ if (i < gimple_call_num_args (call))
+ {
+ tree rhs_type = TREE_TYPE (gimple_call_arg (call, i));
+ if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (rhs_type)))
+ {
+ rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
+ if (rhs < lhs)
+ scalar_type = rhs_type;
+ }
+ }
+ }
*lhs_size_unit = lhs;
*rhs_size_unit = rhs;
tested at run-time. Return TRUE if DDR was successfully inserted.
Return false if versioning is not supported. */
-static bool
+static opt_result
vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- if ((unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS) == 0)
- return false;
+ if ((unsigned) param_vect_max_version_for_alias_checks == 0)
+ return opt_result::failure_at (vect_location,
+ "will not create alias checks, as"
+ " --param vect-max-version-for-alias-checks"
+ " == 0\n");
- if (!runtime_alias_check_p (ddr, loop,
- optimize_loop_nest_for_speed_p (loop)))
- return false;
+ opt_result res
+ = runtime_alias_check_p (ddr, loop,
+ optimize_loop_nest_for_speed_p (loop));
+ if (!res)
+ return res;
LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo).safe_push (ddr);
- return true;
+ return opt_result::success ();
}
/* Record that loop LOOP_VINFO needs to check that VALUE is nonzero. */
return;
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "need run-time check that ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, value);
- dump_printf (MSG_NOTE, " is nonzero\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "need run-time check that %T is nonzero\n",
+ value);
LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value);
}
-/* Return true if we know that the order of vectorized STMTINFO_A and
- vectorized STMTINFO_B will be the same as the order of STMTINFO_A and
- STMTINFO_B. At least one of the statements is a write. */
+/* Return true if we know that the order of vectorized DR_INFO_A and
+ vectorized DR_INFO_B will be the same as the order of DR_INFO_A and
+ DR_INFO_B. At least one of the accesses is a write. */
static bool
-vect_preserves_scalar_order_p (stmt_vec_info stmtinfo_a,
- stmt_vec_info stmtinfo_b)
+vect_preserves_scalar_order_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b)
{
+ stmt_vec_info stmtinfo_a = dr_info_a->stmt;
+ stmt_vec_info stmtinfo_b = dr_info_b->stmt;
+
/* Single statements are always kept in their original order. */
if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
&& !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
return true;
/* STMT_A and STMT_B belong to overlapping groups. All loads in a
- group are emitted at the position of the first scalar load and all
- stores in a group are emitted at the position of the last scalar store.
- Thus writes will happen no earlier than their current position
- (but could happen later) while reads will happen no later than their
- current position (but could happen earlier). Reordering is therefore
- only possible if the first access is a write. */
- if (is_pattern_stmt_p (stmtinfo_a))
- stmtinfo_a = STMT_VINFO_RELATED_STMT (stmtinfo_a);
- if (is_pattern_stmt_p (stmtinfo_b))
- stmtinfo_b = STMT_VINFO_RELATED_STMT (stmtinfo_b);
- stmt_vec_info earlier_stmt_info = get_earlier_stmt (stmtinfo_a, stmtinfo_b);
- return !DR_IS_WRITE (STMT_VINFO_DATA_REF (earlier_stmt_info));
+ SLP group are emitted at the position of the last scalar load and
+ all loads in an interleaving group are emitted at the position
+ of the first scalar load.
+ Stores in a group are emitted at the position of the last scalar store.
+ Compute that position and check whether the resulting order matches
+ the current one.
+ We have not yet decided between SLP and interleaving so we have
+ to conservatively assume both. */
+ stmt_vec_info il_a;
+ stmt_vec_info last_a = il_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
+ if (last_a)
+ {
+ for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_a); s;
+ s = DR_GROUP_NEXT_ELEMENT (s))
+ last_a = get_later_stmt (last_a, s);
+ if (!DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_a)))
+ {
+ for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
+ s = DR_GROUP_NEXT_ELEMENT (s))
+ if (get_later_stmt (il_a, s) == il_a)
+ il_a = s;
+ }
+ else
+ il_a = last_a;
+ }
+ else
+ last_a = il_a = stmtinfo_a;
+ stmt_vec_info il_b;
+ stmt_vec_info last_b = il_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
+ if (last_b)
+ {
+ for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (last_b); s;
+ s = DR_GROUP_NEXT_ELEMENT (s))
+ last_b = get_later_stmt (last_b, s);
+ if (!DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_b)))
+ {
+ for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
+ s = DR_GROUP_NEXT_ELEMENT (s))
+ if (get_later_stmt (il_b, s) == il_b)
+ il_b = s;
+ }
+ else
+ il_b = last_b;
+ }
+ else
+ last_b = il_b = stmtinfo_b;
+ bool a_after_b = (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a);
+ return (/* SLP */
+ (get_later_stmt (last_a, last_b) == last_a) == a_after_b
+ /* Interleaving */
+ && (get_later_stmt (il_a, il_b) == il_a) == a_after_b
+ /* Mixed */
+ && (get_later_stmt (il_a, last_b) == il_a) == a_after_b
+ && (get_later_stmt (last_a, il_b) == last_a) == a_after_b);
}
/* A subroutine of vect_analyze_data_ref_dependence. Handle
loop_vec_info loop_vinfo,
int loop_depth, unsigned int *max_vf)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
lambda_vector dist_v;
unsigned int i;
FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
Note that the alias checks will be removed if the VF ends up
being small enough. */
- return (!STMT_VINFO_GATHER_SCATTER_P
- (vinfo_for_stmt (DR_STMT (DDR_A (ddr))))
- && !STMT_VINFO_GATHER_SCATTER_P
- (vinfo_for_stmt (DR_STMT (DDR_B (ddr))))
+ dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
+ dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
+ return (!STMT_VINFO_GATHER_SCATTER_P (dr_info_a->stmt)
+ && !STMT_VINFO_GATHER_SCATTER_P (dr_info_b->stmt)
&& vect_mark_for_runtime_alias_test (ddr, loop_vinfo));
}
}
/* Function vect_analyze_data_ref_dependence.
- Return TRUE if there (might) exist a dependence between a memory-reference
+ FIXME: I needed to change the sense of the returned flag.
+
+ Return FALSE if there (might) exist a dependence between a memory-reference
DRA and a memory-reference DRB. When versioning for alias may check a
- dependence at run-time, return FALSE. Adjust *MAX_VF according to
+ dependence at run-time, return TRUE. Adjust *MAX_VF according to
the data dependence. */
-static bool
+static opt_result
vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
loop_vec_info loop_vinfo,
unsigned int *max_vf)
{
unsigned int i;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (dra);
+ dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (drb);
+ stmt_vec_info stmtinfo_a = dr_info_a->stmt;
+ stmt_vec_info stmtinfo_b = dr_info_b->stmt;
lambda_vector dist_v;
unsigned int loop_depth;
/* Independent data accesses. */
if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
- return false;
+ return opt_result::success ();
if (dra == drb
|| (DR_IS_READ (dra) && DR_IS_READ (drb)))
- return false;
+ return opt_result::success ();
/* We do not have to consider dependences between accesses that belong
to the same group, unless the stride could be smaller than the
&& (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
== DR_GROUP_FIRST_ELEMENT (stmtinfo_b))
&& !STMT_VINFO_STRIDED_P (stmtinfo_a))
- return false;
+ return opt_result::success ();
/* Even if we have an anti-dependence then, as the vectorized loop covers at
least two scalar iterations, there is always also a true dependence.
|| (DR_IS_WRITE (dra) && DR_IS_READ (drb)))
&& !alias_sets_conflict_p (get_alias_set (DR_REF (dra)),
get_alias_set (DR_REF (drb))))
- return false;
+ return opt_result::success ();
/* Unknown data dependence. */
if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
if ((unsigned int) loop->safelen < *max_vf)
*max_vf = loop->safelen;
LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
- return false;
+ return opt_result::success ();
}
if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "versioning for alias not supported for: "
- "can't determine dependence between ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (dra));
- dump_printf (MSG_MISSED_OPTIMIZATION, " and ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
- return true;
- }
+ return opt_result::failure_at
+ (stmtinfo_a->stmt,
+ "versioning for alias not supported for: "
+ "can't determine dependence between %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "versioning for alias required: "
- "can't determine dependence between ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (dra));
- dump_printf (MSG_MISSED_OPTIMIZATION, " and ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
+ "versioning for alias required: "
+ "can't determine dependence between %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
/* Add to list of ddrs that need to be tested at run-time. */
- return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
+ return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
}
/* Known data dependence. */
if ((unsigned int) loop->safelen < *max_vf)
*max_vf = loop->safelen;
LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
- return false;
+ return opt_result::success ();
}
if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "versioning for alias not supported for: "
- "bad dist vector for ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (dra));
- dump_printf (MSG_MISSED_OPTIMIZATION, " and ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
- return true;
- }
+ return opt_result::failure_at
+ (stmtinfo_a->stmt,
+ "versioning for alias not supported for: "
+ "bad dist vector for %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "versioning for alias required: "
- "bad dist vector for ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_MISSED_OPTIMIZATION, " and ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
+ "versioning for alias required: "
+ "bad dist vector for %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
/* Add to list of ddrs that need to be tested at run-time. */
- return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
+ return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
}
loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
if (DDR_COULD_BE_INDEPENDENT_P (ddr)
&& vect_analyze_possibly_independent_ddr (ddr, loop_vinfo,
loop_depth, max_vf))
- return false;
+ return opt_result::success ();
FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
{
if (dist == 0)
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "dependence distance == 0 between ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "dependence distance == 0 between %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
/* When we perform grouped accesses and perform implicit CSE
by detecting equal accesses and doing disambiguation with
... = a[i];
a[i+1] = ...;
where loads from the group interleave with the store. */
- if (!vect_preserves_scalar_order_p (stmtinfo_a, stmtinfo_b))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "READ_WRITE dependence in interleaving.\n");
- return true;
- }
+ if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
+ return opt_result::failure_at (stmtinfo_a->stmt,
+ "READ_WRITE dependence"
+ " in interleaving.\n");
if (loop->safelen < 2)
{
tree indicator = dr_zero_step_indicator (dra);
if (!indicator || integer_zerop (indicator))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "access also has a zero step\n");
- return true;
- }
+ return opt_result::failure_at (stmtinfo_a->stmt,
+ "access also has a zero step\n");
else if (TREE_CODE (indicator) != INTEGER_CST)
vect_check_nonzero_value (loop_vinfo, indicator);
}
reversed (to make distance vector positive), and the actual
distance is negative. */
if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ dump_printf_loc (MSG_NOTE, vect_location,
"dependence distance negative.\n");
+ /* When doing outer loop vectorization, we need to check if there is
+ a backward dependence at the inner loop level if the dependence
+ at the outer loop is reversed. See PR81740. */
+ if (nested_in_vect_loop_p (loop, stmtinfo_a)
+ || nested_in_vect_loop_p (loop, stmtinfo_b))
+ {
+ unsigned inner_depth = index_in_loop_nest (loop->inner->num,
+ DDR_LOOP_NEST (ddr));
+ if (dist_v[inner_depth] < 0)
+ return opt_result::failure_at (stmtinfo_a->stmt,
+ "not vectorized, dependence "
+ "between data-refs %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
+ }
/* Record a negative dependence distance to later limit the
amount of stmt copying / unrolling we can perform.
Only need to handle read-after-write dependence. */
{
/* The dependence distance requires reduction of the maximal
vectorization factor. */
- *max_vf = abs (dist);
+ *max_vf = abs_dist;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"adjusting maximal vectorization factor to %i\n",
continue;
}
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized, possible dependence "
- "between data-refs ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_NOTE, "\n");
- }
-
- return true;
+ return opt_result::failure_at (stmtinfo_a->stmt,
+ "not vectorized, possible dependence "
+ "between data-refs %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
}
- return false;
+ return opt_result::success ();
}
/* Function vect_analyze_data_ref_dependences.
exist any data dependences between them. Set *MAX_VF according to
the maximum vectorization factor the data dependences allow. */
-bool
+opt_result
vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
unsigned int *max_vf)
{
*max_vf = LOOP_VINFO_ORIG_MAX_VECT_FACTOR (loop_vinfo);
else
FOR_EACH_VEC_ELT (LOOP_VINFO_DDRS (loop_vinfo), i, ddr)
- if (vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf))
- return false;
+ {
+ opt_result res
+ = vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf);
+ if (!res)
+ return res;
+ }
- return true;
+ return opt_result::success ();
}
/* Function vect_slp_analyze_data_ref_dependence.
Return TRUE if there (might) exist a dependence between a memory-reference
- DRA and a memory-reference DRB. When versioning for alias may check a
- dependence at run-time, return FALSE. Adjust *MAX_VF according to
- the data dependence. */
+ DRA and a memory-reference DRB for VINFO. When versioning for alias
+ may check a dependence at run-time, return FALSE. Adjust *MAX_VF
+ according to the data dependence. */
static bool
-vect_slp_analyze_data_ref_dependence (struct data_dependence_relation *ddr)
+vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
+ struct data_dependence_relation *ddr)
{
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
+ dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
+ dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
/* We need to check dependences of statements marked as unvectorizable
as well, they still can prohibit vectorization. */
/* If dra and drb are part of the same interleaving chain consider
them independent. */
- if (STMT_VINFO_GROUPED_ACCESS (vect_dr_stmt (dra))
- && (DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (dra))
- == DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (drb))))
+ if (STMT_VINFO_GROUPED_ACCESS (dr_info_a->stmt)
+ && (DR_GROUP_FIRST_ELEMENT (dr_info_a->stmt)
+ == DR_GROUP_FIRST_ELEMENT (dr_info_b->stmt)))
return false;
/* Unknown data dependence. */
if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't determine dependence between ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_MISSED_OPTIMIZATION, " and ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't determine dependence between %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
}
else if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "determined dependence between ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "determined dependence between %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
return true;
}
disambiguating the loads. */
static bool
-vect_slp_analyze_node_dependences (slp_instance instance, slp_tree node,
+vect_slp_analyze_node_dependences (vec_info *vinfo, slp_tree node,
vec<stmt_vec_info> stores,
stmt_vec_info last_store_info)
{
in NODE verifying we can sink them up to the last stmt in the
group. */
stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
- vec_info *vinfo = last_access_info->vinfo;
- for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
+ for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
{
stmt_vec_info access_info = SLP_TREE_SCALAR_STMTS (node)[k];
if (access_info == last_access_info)
data_reference *store_dr = STMT_VINFO_DATA_REF (store_info);
ddr_p ddr = initialize_data_dependence_relation
(dr_a, store_dr, vNULL);
- dependent = vect_slp_analyze_data_ref_dependence (ddr);
+ dependent
+ = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
free_dependence_relation (ddr);
if (dependent)
break;
{
ddr_p ddr = initialize_data_dependence_relation (dr_a,
dr_b, vNULL);
- dependent = vect_slp_analyze_data_ref_dependence (ddr);
+ dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
free_dependence_relation (ddr);
}
if (dependent)
the maximum vectorization factor the data dependences allow. */
bool
-vect_slp_analyze_instance_dependence (slp_instance instance)
+vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
stmt_vec_info last_store_info = NULL;
if (store)
{
- if (! vect_slp_analyze_node_dependences (instance, store, vNULL, NULL))
+ if (! vect_slp_analyze_node_dependences (vinfo, store, vNULL, NULL))
return false;
/* Mark stores in this instance and remember the last one. */
last_store_info = vect_find_last_scalar_stmt_in_slp (store);
- for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
+ for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, true);
}
slp_tree load;
unsigned int i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, load)
- if (! vect_slp_analyze_node_dependences (instance, load,
+ if (! vect_slp_analyze_node_dependences (vinfo, load,
store
? SLP_TREE_SCALAR_STMTS (store)
: vNULL, last_store_info))
/* Unset the visited flag. */
if (store)
- for (unsigned k = 0; k < SLP_INSTANCE_GROUP_SIZE (instance); ++k)
+ for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, false);
return res;
in STMT_INFO. */
static void
-vect_record_base_alignment (stmt_vec_info stmt_info,
+vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
innermost_loop_behavior *drb)
{
- vec_info *vinfo = stmt_info->vinfo;
bool existed;
innermost_loop_behavior *&entry
= vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
{
entry = drb;
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "recording new base alignment for ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, drb->base_address);
- dump_printf (MSG_NOTE, "\n");
- dump_printf_loc (MSG_NOTE, vect_location,
- " alignment: %d\n", drb->base_alignment);
- dump_printf_loc (MSG_NOTE, vect_location,
- " misalignment: %d\n", drb->base_misalignment);
- dump_printf_loc (MSG_NOTE, vect_location,
- " based on: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "recording new base alignment for %T\n"
+ " alignment: %d\n"
+ " misalignment: %d\n"
+ " based on: %G",
+ drb->base_address,
+ drb->base_alignment,
+ drb->base_misalignment,
+ stmt_info->stmt);
}
}
vect_record_base_alignments (vec_info *vinfo)
{
loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
- struct loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
+ class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
data_reference *dr;
unsigned int i;
FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
if (!DR_IS_CONDITIONAL_IN_STMT (dr)
&& STMT_VINFO_VECTORIZABLE (stmt_info)
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
{
- vect_record_base_alignment (stmt_info, &DR_INNERMOST (dr));
+ vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
/* If DR is nested in the loop that is being vectorized, we can also
record the alignment of the base wrt the outer loop. */
if (loop && nested_in_vect_loop_p (loop, stmt_info))
vect_record_base_alignment
- (stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
+ (vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
}
}
}
-/* Return the target alignment for the vectorized form of DR. */
+/* Return the target alignment for the vectorized form of DR_INFO. */
-static unsigned int
-vect_calculate_target_alignment (struct data_reference *dr)
+static poly_uint64
+vect_calculate_target_alignment (dr_vec_info *dr_info)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
return targetm.vectorize.preferred_vector_alignment (vectype);
}
/* Function vect_compute_data_ref_alignment
- Compute the misalignment of the data reference DR.
+ Compute the misalignment of the data reference DR_INFO.
Output:
- 1. DR_MISALIGNMENT (DR) is defined.
+ 1. DR_MISALIGNMENT (DR_INFO) is defined.
FOR NOW: No analysis is actually performed. Misalignment is calculated
only for trivial cases. TODO. */
static void
-vect_compute_data_ref_alignment (struct data_reference *dr)
+vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- vec_base_alignments *base_alignments = &stmt_info->vinfo->base_alignments;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = NULL;
- tree ref = DR_REF (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
+ vec_base_alignments *base_alignments = &vinfo->base_alignments;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ class loop *loop = NULL;
+ tree ref = DR_REF (dr_info->dr);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
if (dump_enabled_p ())
loop = LOOP_VINFO_LOOP (loop_vinfo);
/* Initialize misalignment to unknown. */
- SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN);
+ SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
return;
- innermost_loop_behavior *drb = vect_dr_behavior (dr);
+ innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
bool step_preserves_misalignment_p;
- unsigned HOST_WIDE_INT vector_alignment
- = vect_calculate_target_alignment (dr) / BITS_PER_UNIT;
- DR_TARGET_ALIGNMENT (dr) = vector_alignment;
+ poly_uint64 vector_alignment
+ = exact_div (vect_calculate_target_alignment (dr_info), BITS_PER_UNIT);
+ DR_TARGET_ALIGNMENT (dr_info) = vector_alignment;
+
+ /* If the main loop has peeled for alignment we have no way of knowing
+ whether the data accesses in the epilogues are aligned. We can't at
+ compile time answer the question whether we have entered the main loop or
+ not. Fixes PR 92351. */
+ if (loop_vinfo)
+ {
+ loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
+ if (orig_loop_vinfo
+ && LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo) != 0)
+ return;
+ }
+
+ unsigned HOST_WIDE_INT vect_align_c;
+ if (!vector_alignment.is_constant (&vect_align_c))
+ return;
/* No step for BB vectorization. */
if (!loop)
else if (nested_in_vect_loop_p (loop, stmt_info))
{
step_preserves_misalignment_p
- = (DR_STEP_ALIGNMENT (dr) % vector_alignment) == 0;
+ = (DR_STEP_ALIGNMENT (dr_info->dr) % vect_align_c) == 0;
if (dump_enabled_p ())
{
{
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
step_preserves_misalignment_p
- = multiple_p (DR_STEP_ALIGNMENT (dr) * vf, vector_alignment);
+ = multiple_p (DR_STEP_ALIGNMENT (dr_info->dr) * vf, vect_align_c);
if (!step_preserves_misalignment_p && dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
base_misalignment = (*entry)->base_misalignment;
}
- if (drb->offset_alignment < vector_alignment
+ if (drb->offset_alignment < vect_align_c
|| !step_preserves_misalignment_p
/* We need to know whether the step wrt the vectorized loop is
negative when computing the starting misalignment below. */
|| TREE_CODE (drb->step) != INTEGER_CST)
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Unknown alignment for access: ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, ref);
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Unknown alignment for access: %T\n", ref);
return;
}
- if (base_alignment < vector_alignment)
+ if (base_alignment < vect_align_c)
{
unsigned int max_alignment;
tree base = get_base_for_alignment (drb->base_address, &max_alignment);
- if (max_alignment < vector_alignment
+ if (max_alignment < vect_align_c
|| !vect_can_force_dr_alignment_p (base,
- vector_alignment * BITS_PER_UNIT))
+ vect_align_c * BITS_PER_UNIT))
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "can't force alignment of ref: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, ref);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "can't force alignment of ref: %T\n", ref);
return;
}
NOTE: This is the only change to the code we make during
the analysis phase, before deciding to vectorize the loop. */
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "force alignment of ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, ref);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "force alignment of %T\n", ref);
- DR_VECT_AUX (dr)->base_decl = base;
- DR_VECT_AUX (dr)->base_misaligned = true;
+ dr_info->base_decl = base;
+ dr_info->base_misaligned = true;
base_misalignment = 0;
}
poly_int64 misalignment
* TREE_INT_CST_LOW (drb->step));
unsigned int const_misalignment;
- if (!known_misalignment (misalignment, vector_alignment,
- &const_misalignment))
+ if (!known_misalignment (misalignment, vect_align_c, &const_misalignment))
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Non-constant misalignment for access: ");
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, ref);
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "Non-constant misalignment for access: %T\n", ref);
return;
}
- SET_DR_MISALIGNMENT (dr, const_misalignment);
+ SET_DR_MISALIGNMENT (dr_info, const_misalignment);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "misalign = %d bytes of ref ", DR_MISALIGNMENT (dr));
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, ref);
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "misalign = %d bytes of ref %T\n",
+ DR_MISALIGNMENT (dr_info), ref);
return;
}
/* Function vect_update_misalignment_for_peel.
- Sets DR's misalignment
- - to 0 if it has the same alignment as DR_PEEL,
- - to the misalignment computed using NPEEL if DR's salignment is known,
+ Sets DR_INFO's misalignment
+ - to 0 if it has the same alignment as DR_PEEL_INFO,
+ - to the misalignment computed using NPEEL if DR_INFO's salignment is known,
- to -1 (unknown) otherwise.
- DR - the data reference whose misalignment is to be adjusted.
- DR_PEEL - the data reference whose misalignment is being made
- zero in the vector loop by the peel.
+ DR_INFO - the data reference whose misalignment is to be adjusted.
+ DR_PEEL_INFO - the data reference whose misalignment is being made
+ zero in the vector loop by the peel.
NPEEL - the number of iterations in the peel loop if the misalignment
- of DR_PEEL is known at compile time. */
+ of DR_PEEL_INFO is known at compile time. */
static void
-vect_update_misalignment_for_peel (struct data_reference *dr,
- struct data_reference *dr_peel, int npeel)
+vect_update_misalignment_for_peel (dr_vec_info *dr_info,
+ dr_vec_info *dr_peel_info, int npeel)
{
unsigned int i;
vec<dr_p> same_aligned_drs;
struct data_reference *current_dr;
- int dr_size = vect_get_scalar_dr_size (dr);
- int dr_peel_size = vect_get_scalar_dr_size (dr_peel);
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- stmt_vec_info peel_stmt_info = vect_dr_stmt (dr_peel);
-
- /* For interleaved data accesses the step in the loop must be multiplied by
- the size of the interleaving group. */
- if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
- dr_size *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
- if (STMT_VINFO_GROUPED_ACCESS (peel_stmt_info))
- dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info);
+ stmt_vec_info peel_stmt_info = dr_peel_info->stmt;
- /* It can be assumed that the data refs with the same alignment as dr_peel
- are aligned in the vector loop. */
- same_aligned_drs = STMT_VINFO_SAME_ALIGN_REFS (vect_dr_stmt (dr_peel));
+ /* It can be assumed that if dr_info has the same alignment as dr_peel,
+ it is aligned in the vector loop. */
+ same_aligned_drs = STMT_VINFO_SAME_ALIGN_REFS (peel_stmt_info);
FOR_EACH_VEC_ELT (same_aligned_drs, i, current_dr)
{
- if (current_dr != dr)
+ if (current_dr != dr_info->dr)
continue;
- gcc_assert (!known_alignment_for_access_p (dr)
- || !known_alignment_for_access_p (dr_peel)
- || (DR_MISALIGNMENT (dr) / dr_size
- == DR_MISALIGNMENT (dr_peel) / dr_peel_size));
- SET_DR_MISALIGNMENT (dr, 0);
+ gcc_assert (!known_alignment_for_access_p (dr_info)
+ || !known_alignment_for_access_p (dr_peel_info)
+ || (DR_MISALIGNMENT (dr_info)
+ == DR_MISALIGNMENT (dr_peel_info)));
+ SET_DR_MISALIGNMENT (dr_info, 0);
return;
}
- if (known_alignment_for_access_p (dr)
- && known_alignment_for_access_p (dr_peel))
+ unsigned HOST_WIDE_INT alignment;
+ if (DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)
+ && known_alignment_for_access_p (dr_info)
+ && known_alignment_for_access_p (dr_peel_info))
{
- bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
- int misal = DR_MISALIGNMENT (dr);
- misal += negative ? -npeel * dr_size : npeel * dr_size;
- misal &= DR_TARGET_ALIGNMENT (dr) - 1;
- SET_DR_MISALIGNMENT (dr, misal);
+ int misal = DR_MISALIGNMENT (dr_info);
+ misal += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
+ misal &= alignment - 1;
+ SET_DR_MISALIGNMENT (dr_info, misal);
return;
}
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "Setting misalignment " \
"to unknown (-1).\n");
- SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN);
+ SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
}
/* Function verify_data_ref_alignment
- Return TRUE if DR can be handled with respect to alignment. */
+ Return TRUE if DR_INFO can be handled with respect to alignment. */
-static bool
-verify_data_ref_alignment (data_reference_p dr)
+static opt_result
+verify_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info)
{
enum dr_alignment_support supportable_dr_alignment
- = vect_supportable_dr_alignment (dr, false);
+ = vect_supportable_dr_alignment (vinfo, dr_info, false);
if (!supportable_dr_alignment)
- {
- if (dump_enabled_p ())
- {
- if (DR_IS_READ (dr))
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: unsupported unaligned load.");
- else
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: unsupported unaligned "
- "store.");
-
- dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- DR_REF (dr));
- dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
- }
- return false;
- }
+ return opt_result::failure_at
+ (dr_info->stmt->stmt,
+ DR_IS_READ (dr_info->dr)
+ ? "not vectorized: unsupported unaligned load: %T\n"
+ : "not vectorized: unsupported unaligned store: %T\n",
+ DR_REF (dr_info->dr));
if (supportable_dr_alignment != dr_aligned && dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"Vectorizing an unaligned access.\n");
- return true;
+ return opt_result::success ();
}
/* Function vect_verify_datarefs_alignment
Return TRUE if all data references in the loop can be
handled with respect to alignment. */
-bool
+opt_result
vect_verify_datarefs_alignment (loop_vec_info vinfo)
{
vec<data_reference_p> datarefs = vinfo->shared->datarefs;
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- if (! verify_data_ref_alignment (dr))
- return false;
+ opt_result res = verify_data_ref_alignment (vinfo, dr_info);
+ if (!res)
+ return res;
}
- return true;
+ return opt_result::success ();
}
/* Given an memory reference EXP return whether its alignment is less
/* Function vector_alignment_reachable_p
- Return true if vector alignment for DR is reachable by peeling
+ Return true if vector alignment for DR_INFO is reachable by peeling
a few loop iterations. Return false otherwise. */
static bool
-vector_alignment_reachable_p (struct data_reference *dr)
+vector_alignment_reachable_p (dr_vec_info *dr_info)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
int elem_size, mis_in_elements;
/* FORNOW: handle only known alignment. */
- if (!known_alignment_for_access_p (dr))
+ if (!known_alignment_for_access_p (dr_info))
return false;
poly_uint64 nelements = TYPE_VECTOR_SUBPARTS (vectype);
poly_uint64 vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
elem_size = vector_element_size (vector_size, nelements);
- mis_in_elements = DR_MISALIGNMENT (dr) / elem_size;
+ mis_in_elements = DR_MISALIGNMENT (dr_info) / elem_size;
if (!multiple_p (nelements - mis_in_elements, DR_GROUP_SIZE (stmt_info)))
return false;
/* If misalignment is known at the compile time then allow peeling
only if natural alignment is reachable through peeling. */
- if (known_alignment_for_access_p (dr) && !aligned_access_p (dr))
+ if (known_alignment_for_access_p (dr_info) && !aligned_access_p (dr_info))
{
HOST_WIDE_INT elmsize =
int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
- "data size =" HOST_WIDE_INT_PRINT_DEC, elmsize);
- dump_printf (MSG_NOTE,
- ". misalignment = %d.\n", DR_MISALIGNMENT (dr));
+ "data size = %wd. misalignment = %d.\n", elmsize,
+ DR_MISALIGNMENT (dr_info));
}
- if (DR_MISALIGNMENT (dr) % elmsize)
+ if (DR_MISALIGNMENT (dr_info) % elmsize)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
}
}
- if (!known_alignment_for_access_p (dr))
+ if (!known_alignment_for_access_p (dr_info))
{
- tree type = TREE_TYPE (DR_REF (dr));
- bool is_packed = not_size_aligned (DR_REF (dr));
+ tree type = TREE_TYPE (DR_REF (dr_info->dr));
+ bool is_packed = not_size_aligned (DR_REF (dr_info->dr));
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
"Unknown misalignment, %snaturally aligned\n",
}
-/* Calculate the cost of the memory access represented by DR. */
+/* Calculate the cost of the memory access represented by DR_INFO. */
static void
-vect_get_data_access_cost (struct data_reference *dr,
+vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
unsigned int *inside_cost,
unsigned int *outside_cost,
stmt_vector_for_cost *body_cost_vec,
stmt_vector_for_cost *prologue_cost_vec)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ stmt_vec_info stmt_info = dr_info->stmt;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
int ncopies;
if (PURE_SLP_STMT (stmt_info))
else
ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
- if (DR_IS_READ (dr))
- vect_get_load_cost (stmt_info, ncopies, true, inside_cost, outside_cost,
- prologue_cost_vec, body_cost_vec, false);
+ if (DR_IS_READ (dr_info->dr))
+ vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost,
+ outside_cost, prologue_cost_vec, body_cost_vec, false);
else
- vect_get_store_cost (stmt_info, ncopies, inside_cost, body_cost_vec);
+ vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
typedef struct _vect_peel_info
{
- struct data_reference *dr;
+ dr_vec_info *dr_info;
int npeel;
unsigned int count;
} *vect_peel_info;
typedef struct _vect_peel_extended_info
{
+ vec_info *vinfo;
struct _vect_peel_info peel_info;
unsigned int inside_cost;
unsigned int outside_cost;
}
-/* Insert DR into peeling hash table with NPEEL as key. */
+/* Insert DR_INFO into peeling hash table with NPEEL as key. */
static void
vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
- loop_vec_info loop_vinfo, struct data_reference *dr,
+ loop_vec_info loop_vinfo, dr_vec_info *dr_info,
int npeel)
{
struct _vect_peel_info elem, *slot;
_vect_peel_info **new_slot;
- bool supportable_dr_alignment = vect_supportable_dr_alignment (dr, true);
+ bool supportable_dr_alignment
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
elem.npeel = npeel;
slot = peeling_htab->find (&elem);
{
slot = XNEW (struct _vect_peel_info);
slot->npeel = npeel;
- slot->dr = dr;
+ slot->dr_info = dr_info;
slot->count = 1;
new_slot = peeling_htab->find_slot (slot, INSERT);
*new_slot = slot;
{
max->peel_info.npeel = elem->npeel;
max->peel_info.count = elem->count;
- max->peel_info.dr = elem->dr;
+ max->peel_info.dr_info = elem->dr_info;
}
return 1;
}
-/* Get the costs of peeling NPEEL iterations checking data access costs
- for all data refs. If UNKNOWN_MISALIGNMENT is true, we assume DR0's
- misalignment will be zero after peeling. */
+/* Get the costs of peeling NPEEL iterations for LOOP_VINFO, checking
+ data access costs for all data refs. If UNKNOWN_MISALIGNMENT is true,
+ we assume DR0_INFO's misalignment will be zero after peeling. */
static void
-vect_get_peeling_costs_all_drs (vec<data_reference_p> datarefs,
- struct data_reference *dr0,
+vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
+ dr_vec_info *dr0_info,
unsigned int *inside_cost,
unsigned int *outside_cost,
stmt_vector_for_cost *body_cost_vec,
unsigned int npeel,
bool unknown_misalignment)
{
+ vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
unsigned i;
data_reference *dr;
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
continue;
int save_misalignment;
- save_misalignment = DR_MISALIGNMENT (dr);
+ save_misalignment = DR_MISALIGNMENT (dr_info);
if (npeel == 0)
;
- else if (unknown_misalignment && dr == dr0)
- SET_DR_MISALIGNMENT (dr, 0);
+ else if (unknown_misalignment && dr_info == dr0_info)
+ SET_DR_MISALIGNMENT (dr_info, 0);
else
- vect_update_misalignment_for_peel (dr, dr0, npeel);
- vect_get_data_access_cost (dr, inside_cost, outside_cost,
+ vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
+ vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost,
body_cost_vec, prologue_cost_vec);
- SET_DR_MISALIGNMENT (dr, save_misalignment);
+ SET_DR_MISALIGNMENT (dr_info, save_misalignment);
}
}
vect_peel_info elem = *slot;
int dummy;
unsigned int inside_cost = 0, outside_cost = 0;
- stmt_vec_info stmt_info = vect_dr_stmt (elem->dr);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
stmt_vector_for_cost prologue_cost_vec, body_cost_vec,
epilogue_cost_vec;
body_cost_vec.create (2);
epilogue_cost_vec.create (2);
- vect_get_peeling_costs_all_drs (LOOP_VINFO_DATAREFS (loop_vinfo),
- elem->dr, &inside_cost, &outside_cost,
- &body_cost_vec, &prologue_cost_vec,
- elem->npeel, false);
+ vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost,
+ &outside_cost, &body_cost_vec,
+ &prologue_cost_vec, elem->npeel, false);
body_cost_vec.release ();
{
min->inside_cost = inside_cost;
min->outside_cost = outside_cost;
- min->peel_info.dr = elem->dr;
+ min->peel_info.dr_info = elem->dr_info;
min->peel_info.npeel = elem->npeel;
min->peel_info.count = elem->count;
}
{
struct _vect_peel_extended_info res;
- res.peel_info.dr = NULL;
+ res.peel_info.dr_info = NULL;
+ res.vinfo = loop_vinfo;
if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
{
/* Return true if the new peeling NPEEL is supported. */
static bool
-vect_peeling_supportable (loop_vec_info loop_vinfo, struct data_reference *dr0,
+vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
unsigned npeel)
{
unsigned i;
{
int save_misalignment;
- if (dr == dr0)
+ if (dr == dr0_info->dr)
continue;
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
/* For interleaving, only the alignment of the first access
matters. */
if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- save_misalignment = DR_MISALIGNMENT (dr);
- vect_update_misalignment_for_peel (dr, dr0, npeel);
- supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
- SET_DR_MISALIGNMENT (dr, save_misalignment);
+ save_misalignment = DR_MISALIGNMENT (dr_info);
+ vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
+ supportable_dr_alignment
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
+ SET_DR_MISALIGNMENT (dr_info, save_misalignment);
if (!supportable_dr_alignment)
return false;
(whether to generate regular loads/stores, or with special handling for
misalignment). */
-bool
+opt_result
vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
{
vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
enum dr_alignment_support supportable_dr_alignment;
- struct data_reference *dr0 = NULL, *first_store = NULL;
+ dr_vec_info *first_store = NULL;
+ dr_vec_info *dr0_info = NULL;
struct data_reference *dr;
unsigned int i, j;
bool do_peeling = false;
bool do_versioning = false;
- bool stat;
unsigned int npeel = 0;
bool one_misalignment_known = false;
bool one_misalignment_unknown = false;
bool one_dr_unsupportable = false;
- struct data_reference *unsupportable_dr = NULL;
+ dr_vec_info *unsupportable_dr_info = NULL;
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
unsigned possible_npeel_number = 1;
tree vectype;
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
if (!STMT_VINFO_RELEVANT_P (stmt_info))
continue;
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- supportable_dr_alignment = vect_supportable_dr_alignment (dr, true);
- do_peeling = vector_alignment_reachable_p (dr);
+ supportable_dr_alignment
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, true);
+ do_peeling = vector_alignment_reachable_p (dr_info);
if (do_peeling)
{
- if (known_alignment_for_access_p (dr))
+ if (known_alignment_for_access_p (dr_info))
{
unsigned int npeel_tmp = 0;
bool negative = tree_int_cst_compare (DR_STEP (dr),
size_zero_node) < 0;
vectype = STMT_VINFO_VECTYPE (stmt_info);
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
- unsigned int dr_size = vect_get_scalar_dr_size (dr);
- mis = (negative ? DR_MISALIGNMENT (dr) : -DR_MISALIGNMENT (dr));
- if (DR_MISALIGNMENT (dr) != 0)
+ /* If known_alignment_for_access_p then we have set
+ DR_MISALIGNMENT which is only done if we know it at compiler
+ time, so it is safe to assume target alignment is constant.
+ */
+ unsigned int target_align =
+ DR_TARGET_ALIGNMENT (dr_info).to_constant ();
+ unsigned int dr_size = vect_get_scalar_dr_size (dr_info);
+ mis = (negative
+ ? DR_MISALIGNMENT (dr_info)
+ : -DR_MISALIGNMENT (dr_info));
+ if (DR_MISALIGNMENT (dr_info) != 0)
npeel_tmp = (mis & (target_align - 1)) / dr_size;
/* For multiple types, it is possible that the bigger type access
/* NPEEL_TMP is 0 when there is no misalignment, but also
allow peeling NELEMENTS. */
- if (DR_MISALIGNMENT (dr) == 0)
+ if (DR_MISALIGNMENT (dr_info) == 0)
possible_npeel_number++;
}
for (j = 0; j < possible_npeel_number; j++)
{
vect_peeling_hash_insert (&peeling_htab, loop_vinfo,
- dr, npeel_tmp);
+ dr_info, npeel_tmp);
npeel_tmp += target_align / dr_size;
}
stores over load. */
unsigned same_align_drs
= STMT_VINFO_SAME_ALIGN_REFS (stmt_info).length ();
- if (!dr0
+ if (!dr0_info
|| same_align_drs_max < same_align_drs)
{
same_align_drs_max = same_align_drs;
- dr0 = dr;
+ dr0_info = dr_info;
}
/* For data-refs with the same number of related
accesses prefer the one where the misalign
computation will be invariant in the outermost loop. */
else if (same_align_drs_max == same_align_drs)
{
- struct loop *ivloop0, *ivloop;
+ class loop *ivloop0, *ivloop;
ivloop0 = outermost_invariant_loop_for_expr
- (loop, DR_BASE_ADDRESS (dr0));
+ (loop, DR_BASE_ADDRESS (dr0_info->dr));
ivloop = outermost_invariant_loop_for_expr
(loop, DR_BASE_ADDRESS (dr));
if ((ivloop && !ivloop0)
|| (ivloop && ivloop0
&& flow_loop_nested_p (ivloop, ivloop0)))
- dr0 = dr;
+ dr0_info = dr_info;
}
one_misalignment_unknown = true;
if (!supportable_dr_alignment)
{
one_dr_unsupportable = true;
- unsupportable_dr = dr;
+ unsupportable_dr_info = dr_info;
}
if (!first_store && DR_IS_WRITE (dr))
- first_store = dr;
+ first_store = dr_info;
}
}
else
{
- if (!aligned_access_p (dr))
+ if (!aligned_access_p (dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
stmt_vector_for_cost dummy;
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, dr0,
+ vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info,
&load_inside_cost,
&load_outside_cost,
&dummy, &dummy, estimated_npeels, true);
if (first_store)
{
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, first_store,
+ vect_get_peeling_costs_all_drs (loop_vinfo, first_store,
&store_inside_cost,
&store_outside_cost,
&dummy, &dummy,
|| (load_inside_cost == store_inside_cost
&& load_outside_cost > store_outside_cost))
{
- dr0 = first_store;
+ dr0_info = first_store;
peel_for_unknown_alignment.inside_cost = store_inside_cost;
peel_for_unknown_alignment.outside_cost = store_outside_cost;
}
epilogue_cost_vec.release ();
peel_for_unknown_alignment.peel_info.count = 1
- + STMT_VINFO_SAME_ALIGN_REFS (vect_dr_stmt (dr0)).length ();
+ + STMT_VINFO_SAME_ALIGN_REFS (dr0_info->stmt).length ();
}
peel_for_unknown_alignment.peel_info.npeel = 0;
- peel_for_unknown_alignment.peel_info.dr = dr0;
+ peel_for_unknown_alignment.peel_info.dr_info = dr0_info;
best_peel = peel_for_unknown_alignment;
peel_for_known_alignment.inside_cost = INT_MAX;
peel_for_known_alignment.outside_cost = INT_MAX;
peel_for_known_alignment.peel_info.count = 0;
- peel_for_known_alignment.peel_info.dr = NULL;
+ peel_for_known_alignment.peel_info.dr_info = NULL;
if (do_peeling && one_misalignment_known)
{
}
/* Compare costs of peeling for known and unknown alignment. */
- if (peel_for_known_alignment.peel_info.dr != NULL
+ if (peel_for_known_alignment.peel_info.dr_info != NULL
&& peel_for_unknown_alignment.inside_cost
>= peel_for_known_alignment.inside_cost)
{
since we'd have to discard a chosen peeling except when it accidentally
aligned the unsupportable data ref. */
if (one_dr_unsupportable)
- dr0 = unsupportable_dr;
+ dr0_info = unsupportable_dr_info;
else if (do_peeling)
{
/* Calculate the penalty for no peeling, i.e. leaving everything as-is.
stmt_vector_for_cost dummy;
dummy.create (2);
- vect_get_peeling_costs_all_drs (datarefs, NULL, &nopeel_inside_cost,
+ vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
&nopeel_outside_cost, &dummy, &dummy,
0, false);
dummy.release ();
epilogue_cost_vec.release ();
npeel = best_peel.peel_info.npeel;
- dr0 = best_peel.peel_info.dr;
+ dr0_info = best_peel.peel_info.dr_info;
/* If no peeling is not more expensive than the best peeling we
have so far, don't perform any peeling. */
if (do_peeling)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr0);
+ stmt_vec_info stmt_info = dr0_info->stmt;
vectype = STMT_VINFO_VECTYPE (stmt_info);
- if (known_alignment_for_access_p (dr0))
+ if (known_alignment_for_access_p (dr0_info))
{
- bool negative = tree_int_cst_compare (DR_STEP (dr0),
+ bool negative = tree_int_cst_compare (DR_STEP (dr0_info->dr),
size_zero_node) < 0;
if (!npeel)
{
updating DR_MISALIGNMENT values. The peeling factor is the
vectorization factor minus the misalignment as an element
count. */
- mis = negative ? DR_MISALIGNMENT (dr0) : -DR_MISALIGNMENT (dr0);
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr0);
+ mis = (negative
+ ? DR_MISALIGNMENT (dr0_info)
+ : -DR_MISALIGNMENT (dr0_info));
+ /* If known_alignment_for_access_p then we have set
+ DR_MISALIGNMENT which is only done if we know it at compiler
+ time, so it is safe to assume target alignment is constant.
+ */
+ unsigned int target_align =
+ DR_TARGET_ALIGNMENT (dr0_info).to_constant ();
npeel = ((mis & (target_align - 1))
- / vect_get_scalar_dr_size (dr0));
+ / vect_get_scalar_dr_size (dr0_info));
}
/* For interleaved data access every iteration accesses all the
members of the group, therefore we divide the number of iterations
by the group size. */
- stmt_info = vect_dr_stmt (dr0);
if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
npeel /= DR_GROUP_SIZE (stmt_info);
}
/* Ensure that all datarefs can be vectorized after the peel. */
- if (!vect_peeling_supportable (loop_vinfo, dr0, npeel))
+ if (!vect_peeling_supportable (loop_vinfo, dr0_info, npeel))
do_peeling = false;
/* Check if all datarefs are supportable and log. */
- if (do_peeling && known_alignment_for_access_p (dr0) && npeel == 0)
+ if (do_peeling && known_alignment_for_access_p (dr0_info) && npeel == 0)
{
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ opt_result stat = vect_verify_datarefs_alignment (loop_vinfo);
if (!stat)
do_peeling = false;
else
if (do_peeling)
{
unsigned max_allowed_peel
- = PARAM_VALUE (PARAM_VECT_MAX_PEELING_FOR_ALIGNMENT);
+ = param_vect_max_peeling_for_alignment;
+ if (flag_vect_cost_model == VECT_COST_MODEL_CHEAP)
+ max_allowed_peel = 0;
if (max_allowed_peel != (unsigned)-1)
{
unsigned max_peel = npeel;
if (max_peel == 0)
{
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr0);
- max_peel = target_align / vect_get_scalar_dr_size (dr0) - 1;
+ poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr0_info);
+ unsigned HOST_WIDE_INT target_align_c;
+ if (target_align.is_constant (&target_align_c))
+ max_peel =
+ target_align_c / vect_get_scalar_dr_size (dr0_info) - 1;
+ else
+ {
+ do_peeling = false;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Disable peeling, max peels set and vector"
+ " alignment unknown\n");
+ }
}
if (max_peel > max_allowed_peel)
{
vectorization factor times the size). Otherwise, the
misalignment of DR_i must be set to unknown. */
FOR_EACH_VEC_ELT (datarefs, i, dr)
- if (dr != dr0)
+ if (dr != dr0_info->dr)
{
/* Strided accesses perform only component accesses, alignment
is irrelevant for them. */
- stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ stmt_info = dr_info->stmt;
if (STMT_VINFO_STRIDED_P (stmt_info)
&& !STMT_VINFO_GROUPED_ACCESS (stmt_info))
continue;
- vect_update_misalignment_for_peel (dr, dr0, npeel);
+ vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
}
- LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0;
+ LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
if (npeel)
LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = npeel;
else
LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
- = DR_MISALIGNMENT (dr0);
- SET_DR_MISALIGNMENT (dr0, 0);
+ = DR_MISALIGNMENT (dr0_info);
+ SET_DR_MISALIGNMENT (dr0_info, 0);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
/* The inside-loop cost will be accounted for in vectorizable_load
and vectorizable_store correctly with adjusted alignments.
Drop the body_cst_vec on the floor here. */
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ opt_result stat = vect_verify_datarefs_alignment (loop_vinfo);
gcc_assert (stat);
return stat;
}
/* (2) Versioning to force alignment. */
/* Try versioning if:
- 1) optimize loop for speed
+ 1) optimize loop for speed and the cost-model is not cheap
2) there is at least one unsupported misaligned data ref with an unknown
misalignment, and
3) all misaligned data refs with a known misalignment are supported, and
4) the number of runtime alignment checks is within reason. */
- do_versioning =
- optimize_loop_nest_for_speed_p (loop)
- && (!loop->inner); /* FORNOW */
+ do_versioning
+ = (optimize_loop_nest_for_speed_p (loop)
+ && !loop->inner /* FORNOW */
+ && flag_vect_cost_model != VECT_COST_MODEL_CHEAP);
if (do_versioning)
{
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
/* For interleaving, only the alignment of the first access
matters. */
- if (aligned_access_p (dr)
+ if (aligned_access_p (dr_info)
|| (STMT_VINFO_GROUPED_ACCESS (stmt_info)
&& DR_GROUP_FIRST_ELEMENT (stmt_info) != stmt_info))
continue;
break;
}
- supportable_dr_alignment = vect_supportable_dr_alignment (dr, false);
+ supportable_dr_alignment
+ = vect_supportable_dr_alignment (loop_vinfo, dr_info, false);
if (!supportable_dr_alignment)
{
int mask;
tree vectype;
- if (known_alignment_for_access_p (dr)
+ if (known_alignment_for_access_p (dr_info)
|| LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
- >= (unsigned) PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS))
+ >= (unsigned) param_vect_max_version_for_alignment_checks)
{
do_versioning = false;
break;
}
- stmt_info = vect_dr_stmt (dr);
vectype = STMT_VINFO_VECTYPE (stmt_info);
gcc_assert (vectype);
break;
}
+ /* Forcing alignment in the first iteration is no good if
+ we don't keep it across iterations. For now, just disable
+ versioning in this case.
+ ?? We could actually unroll the loop to achieve the required
+ overall step alignment, and forcing the alignment could be
+ done by doing some iterations of the non-vectorized loop. */
+ if (!multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ * DR_STEP_ALIGNMENT (dr),
+ DR_TARGET_ALIGNMENT (dr_info)))
+ {
+ do_versioning = false;
+ break;
+ }
+
/* The rightmost bits of an aligned address must be zeros.
Construct the mask needed for this test. For example,
GET_MODE_SIZE for the vector mode V4SI is 16 bytes so the
mask must be 15 = 0xf. */
mask = size - 1;
- /* FORNOW: use the same mask to test all potentially unaligned
- references in the loop. The vectorizer currently supports
- a single vector size, see the reference to
- GET_MODE_NUNITS (TYPE_MODE (vectype)) where the
- vectorization factor is computed. */
- gcc_assert (!LOOP_VINFO_PTR_MASK (loop_vinfo)
- || LOOP_VINFO_PTR_MASK (loop_vinfo) == mask);
+ /* FORNOW: use the same mask to test all potentially unaligned
+ references in the loop. */
+ if (LOOP_VINFO_PTR_MASK (loop_vinfo)
+ && LOOP_VINFO_PTR_MASK (loop_vinfo) != mask)
+ {
+ do_versioning = false;
+ break;
+ }
+
LOOP_VINFO_PTR_MASK (loop_vinfo) = mask;
LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).safe_push (stmt_info);
}
of the loop being vectorized. */
FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
{
- dr = STMT_VINFO_DATA_REF (stmt_info);
- SET_DR_MISALIGNMENT (dr, 0);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ SET_DR_MISALIGNMENT (dr_info, 0);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
"Alignment of access forced using versioning.\n");
/* Peeling and versioning can't be done together at this time. */
gcc_assert (! (do_peeling && do_versioning));
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ opt_result stat = vect_verify_datarefs_alignment (loop_vinfo);
gcc_assert (stat);
return stat;
}
/* This point is reached if neither peeling nor versioning is being done. */
gcc_assert (! (do_peeling || do_versioning));
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ opt_result stat = vect_verify_datarefs_alignment (loop_vinfo);
return stat;
}
/* Function vect_find_same_alignment_drs.
- Update group and alignment relations according to the chosen
+ Update group and alignment relations in VINFO according to the chosen
vectorization factor. */
static void
-vect_find_same_alignment_drs (struct data_dependence_relation *ddr)
+vect_find_same_alignment_drs (vec_info *vinfo, data_dependence_relation *ddr)
{
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
+ dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
+ stmt_vec_info stmtinfo_a = dr_info_a->stmt;
+ stmt_vec_info stmtinfo_b = dr_info_b->stmt;
if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
return;
if (maybe_ne (diff, 0))
{
/* Get the wider of the two alignments. */
- unsigned int align_a = (vect_calculate_target_alignment (dra)
- / BITS_PER_UNIT);
- unsigned int align_b = (vect_calculate_target_alignment (drb)
- / BITS_PER_UNIT);
- unsigned int max_align = MAX (align_a, align_b);
+ poly_uint64 align_a =
+ exact_div (vect_calculate_target_alignment (dr_info_a),
+ BITS_PER_UNIT);
+ poly_uint64 align_b =
+ exact_div (vect_calculate_target_alignment (dr_info_b),
+ BITS_PER_UNIT);
+ unsigned HOST_WIDE_INT align_a_c, align_b_c;
+ if (!align_a.is_constant (&align_a_c)
+ || !align_b.is_constant (&align_b_c))
+ return;
+
+ unsigned HOST_WIDE_INT max_align = MAX (align_a_c, align_b_c);
/* Require the gap to be a multiple of the larger vector alignment. */
if (!multiple_p (diff, max_align))
STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_a).safe_push (drb);
STMT_VINFO_SAME_ALIGN_REFS (stmtinfo_b).safe_push (dra);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "accesses have the same alignment: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "accesses have the same alignment: %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
}
Analyze the alignment of the data-references in the loop.
Return FALSE if a data reference is found that cannot be vectorized. */
-bool
+opt_result
vect_analyze_data_refs_alignment (loop_vec_info vinfo)
{
DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment");
unsigned int i;
FOR_EACH_VEC_ELT (ddrs, i, ddr)
- vect_find_same_alignment_drs (ddr);
+ vect_find_same_alignment_drs (vinfo, ddr);
vec<data_reference_p> datarefs = vinfo->shared->datarefs;
struct data_reference *dr;
vect_record_base_alignments (vinfo);
FOR_EACH_VEC_ELT (datarefs, i, dr)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- if (STMT_VINFO_VECTORIZABLE (stmt_info))
- vect_compute_data_ref_alignment (dr);
+ dr_vec_info *dr_info = vinfo->lookup_dr (dr);
+ if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
+ vect_compute_data_ref_alignment (vinfo, dr_info);
}
- return true;
+ return opt_result::success ();
}
/* Analyze alignment of DRs of stmts in NODE. */
static bool
-vect_slp_analyze_and_verify_node_alignment (slp_tree node)
+vect_slp_analyze_and_verify_node_alignment (vec_info *vinfo, slp_tree node)
{
/* We vectorize from the first scalar stmt in the node unless
the node is permuted in which case we start from the first
element in the group. */
stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
- data_reference_p first_dr = STMT_VINFO_DATA_REF (first_stmt_info);
+ dr_vec_info *first_dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
- data_reference_p dr = STMT_VINFO_DATA_REF (first_stmt_info);
- vect_compute_data_ref_alignment (dr);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
+ vect_compute_data_ref_alignment (vinfo, dr_info);
/* For creating the data-ref pointer we need alignment of the
first element anyway. */
- if (dr != first_dr)
- vect_compute_data_ref_alignment (first_dr);
- if (! verify_data_ref_alignment (dr))
+ if (dr_info != first_dr_info)
+ vect_compute_data_ref_alignment (vinfo, first_dr_info);
+ if (! verify_data_ref_alignment (vinfo, dr_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
Return FALSE if a data reference is found that cannot be vectorized. */
bool
-vect_slp_analyze_and_verify_instance_alignment (slp_instance instance)
+vect_slp_analyze_and_verify_instance_alignment (vec_info *vinfo,
+ slp_instance instance)
{
DUMP_VECT_SCOPE ("vect_slp_analyze_and_verify_instance_alignment");
slp_tree node;
unsigned i;
FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
- if (! vect_slp_analyze_and_verify_node_alignment (node))
+ if (! vect_slp_analyze_and_verify_node_alignment (vinfo, node))
return false;
node = SLP_INSTANCE_TREE (instance);
if (STMT_VINFO_DATA_REF (SLP_TREE_SCALAR_STMTS (node)[0])
&& ! vect_slp_analyze_and_verify_node_alignment
- (SLP_INSTANCE_TREE (instance)))
+ (vinfo, SLP_INSTANCE_TREE (instance)))
return false;
return true;
}
-/* Analyze groups of accesses: check that DR belongs to a group of
+/* Analyze groups of accesses: check that DR_INFO belongs to a group of
accesses of legal size, step, etc. Detect gaps, single element
interleaving, and other special cases. Set grouped access info.
Collect groups of strided stores for further use in SLP analysis.
Worker for vect_analyze_group_access. */
static bool
-vect_analyze_group_access_1 (struct data_reference *dr)
+vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
{
+ data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ stmt_vec_info stmt_info = dr_info->stmt;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
HOST_WIDE_INT dr_step = -1;
HOST_WIDE_INT groupsize, last_accessed_element = 1;
bool slp_impossible = false;
if ((dr_step % type_size) != 0)
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "Step ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, step);
- dump_printf (MSG_NOTE,
- " is not a multiple of the element size for ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr));
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Step %T is not a multiple of the element size"
+ " for %T\n",
+ step, DR_REF (dr));
return false;
}
groupsize = absu_hwi (dr_step) / type_size;
DR_GROUP_SIZE (stmt_info) = groupsize;
DR_GROUP_GAP (stmt_info) = groupsize - 1;
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "Detected single element interleaving ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr));
- dump_printf (MSG_NOTE, " step ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, step);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Detected single element interleaving %T"
+ " step %T\n",
+ DR_REF (dr), step);
return true;
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not consecutive access ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not consecutive access %G", stmt_info->stmt);
if (bb_vinfo)
{
/* Mark the statement as unvectorizable. */
- STMT_VINFO_VECTORIZABLE (vect_dr_stmt (dr)) = false;
+ STMT_VINFO_VECTORIZABLE (stmt_info) = false;
return true;
}
- dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
STMT_VINFO_STRIDED_P (stmt_info) = true;
return true;
}
struct data_reference *data_ref = dr;
unsigned int count = 1;
tree prev_init = DR_INIT (data_ref);
- stmt_vec_info prev = stmt_info;
HOST_WIDE_INT diff, gaps = 0;
/* By construction, all group members have INTEGER_CST DR_INITs. */
while (next)
{
- /* Skip same data-refs. In case that two or more stmts share
- data-ref (supported only for loads), we vectorize only the first
- stmt, and the rest get their vectorized loads from the first
- one. */
- if (!tree_int_cst_compare (DR_INIT (data_ref),
- DR_INIT (STMT_VINFO_DATA_REF (next))))
- {
- if (DR_IS_WRITE (data_ref))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Two store stmts share the same dr.\n");
- return false;
- }
-
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "Two or more load stmts share the same dr.\n");
-
- /* For load use the same data-ref load. */
- DR_GROUP_SAME_DR_STMT (next) = prev;
-
- prev = next;
- next = DR_GROUP_NEXT_ELEMENT (next);
- continue;
- }
+ /* We never have the same DR multiple times. */
+ gcc_assert (tree_int_cst_compare (DR_INIT (data_ref),
+ DR_INIT (STMT_VINFO_DATA_REF (next))) != 0);
- prev = next;
data_ref = STMT_VINFO_DATA_REF (next);
/* All group members have the same STEP by construction. */
if (groupsize != count
&& !DR_IS_READ (dr))
{
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "interleaved store with gaps\n");
- return false;
+ groupsize = count;
+ STMT_VINFO_STRIDED_P (stmt_info) = true;
}
/* If there is a gap after the last load in the group it is the
"Detected interleaving ");
if (DR_IS_READ (dr))
dump_printf (MSG_NOTE, "load ");
+ else if (STMT_VINFO_STRIDED_P (stmt_info))
+ dump_printf (MSG_NOTE, "strided store ");
else
dump_printf (MSG_NOTE, "store ");
- dump_printf (MSG_NOTE, "of size %u starting with ",
+ dump_printf (MSG_NOTE, "of size %u\n",
(unsigned)groupsize);
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
+ dump_printf_loc (MSG_NOTE, vect_location, "\t%G", stmt_info->stmt);
+ next = DR_GROUP_NEXT_ELEMENT (stmt_info);
+ while (next)
+ {
+ if (DR_GROUP_GAP (next) != 1)
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "\t<gap of %d elements>\n",
+ DR_GROUP_GAP (next) - 1);
+ dump_printf_loc (MSG_NOTE, vect_location, "\t%G", next->stmt);
+ next = DR_GROUP_NEXT_ELEMENT (next);
+ }
if (DR_GROUP_GAP (stmt_info) != 0)
dump_printf_loc (MSG_NOTE, vect_location,
- "There is a gap of %u elements after the group\n",
+ "\t<gap of %d elements>\n",
DR_GROUP_GAP (stmt_info));
}
return true;
}
-/* Analyze groups of accesses: check that DR belongs to a group of
+/* Analyze groups of accesses: check that DR_INFO belongs to a group of
accesses of legal size, step, etc. Detect gaps, single element
interleaving, and other special cases. Set grouped access info.
Collect groups of strided stores for further use in SLP analysis. */
static bool
-vect_analyze_group_access (struct data_reference *dr)
+vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
{
- if (!vect_analyze_group_access_1 (dr))
+ if (!vect_analyze_group_access_1 (vinfo, dr_info))
{
/* Dissolve the group if present. */
- stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (vect_dr_stmt (dr));
+ stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
while (stmt_info)
{
stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
return true;
}
-/* Analyze the access pattern of the data-reference DR.
+/* Analyze the access pattern of the data-reference DR_INFO.
In case of non-consecutive accesses call vect_analyze_group_access() to
analyze groups of accesses. */
static bool
-vect_analyze_data_ref_access (struct data_reference *dr)
+vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
{
+ data_reference *dr = dr_info->dr;
tree step = DR_STEP (dr);
tree scalar_type = TREE_TYPE (DR_REF (dr));
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = NULL;
+ stmt_vec_info stmt_info = dr_info->stmt;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ class loop *loop = NULL;
if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
return true;
if (TREE_CODE (step) != INTEGER_CST)
return (STMT_VINFO_STRIDED_P (stmt_info)
&& (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
- || vect_analyze_group_access (dr)));
+ || vect_analyze_group_access (vinfo, dr_info)));
/* Not consecutive access - check if it's a part of interleaving group. */
- return vect_analyze_group_access (dr);
+ return vect_analyze_group_access (vinfo, dr_info);
}
/* Compare two data-references DRA and DRB to group them into chunks
}
/* Return true if vectorizable_* routines can handle statements STMT1_INFO
- and STMT2_INFO being in a single group. */
+ and STMT2_INFO being in a single group. When ALLOW_SLP_P, masked loads can
+ be grouped in SLP mode. */
static bool
-can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
+can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info,
+ bool allow_slp_p)
{
if (gimple_assign_single_p (stmt1_info->stmt))
return gimple_assign_single_p (stmt2_info->stmt);
like those created by build_mask_conversion. */
tree mask1 = gimple_call_arg (call1, 2);
tree mask2 = gimple_call_arg (call2, 2);
- if (!operand_equal_p (mask1, mask2, 0))
+ if (!operand_equal_p (mask1, mask2, 0)
+ && (ifn == IFN_MASK_STORE || !allow_slp_p))
{
mask1 = strip_conversion (mask1);
if (!mask1)
FORNOW: handle only arrays and pointer accesses. */
-bool
+opt_result
vect_analyze_data_ref_accesses (vec_info *vinfo)
{
unsigned int i;
DUMP_VECT_SCOPE ("vect_analyze_data_ref_accesses");
if (datarefs.is_empty ())
- return true;
+ return opt_result::success ();
/* Sort the array of datarefs to make building the interleaving chains
linear. Don't modify the original vector's order, it is needed for
determining what dependencies are reversed. */
vec<data_reference_p> datarefs_copy = datarefs.copy ();
datarefs_copy.qsort (dr_group_sort_cmp);
+ hash_set<stmt_vec_info> to_fixup;
/* Build the interleaving chains. */
for (i = 0; i < datarefs_copy.length () - 1;)
{
data_reference_p dra = datarefs_copy[i];
- stmt_vec_info stmtinfo_a = vect_dr_stmt (dra);
+ dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
+ stmt_vec_info stmtinfo_a = dr_info_a->stmt;
stmt_vec_info lastinfo = NULL;
if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a))
for (i = i + 1; i < datarefs_copy.length (); ++i)
{
data_reference_p drb = datarefs_copy[i];
- stmt_vec_info stmtinfo_b = vect_dr_stmt (drb);
+ dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
+ stmt_vec_info stmtinfo_b = dr_info_b->stmt;
if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b)
|| STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
break;
|| data_ref_compare_tree (DR_BASE_ADDRESS (dra),
DR_BASE_ADDRESS (drb)) != 0
|| data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)) != 0
- || !can_group_stmts_p (stmtinfo_a, stmtinfo_b))
+ || !can_group_stmts_p (stmtinfo_a, stmtinfo_b, true))
break;
/* Check that the data-refs have the same constant size. */
|| TREE_CODE (DR_INIT (drb)) != INTEGER_CST)
break;
+ /* Different .GOMP_SIMD_LANE calls still give the same lane,
+ just hold extra information. */
+ if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
+ && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
+ && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
+ break;
+
/* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb). */
HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
{
gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1]))
< gimple_uid (DR_STMT (drb)));
- /* ??? For now we simply "drop" the later reference which is
- otherwise the same rather than finishing off this group.
- In the end we'd want to re-process duplicates forming
- multiple groups from the refs, likely by just collecting
- all candidates (including duplicates and split points
- below) in a vector and then process them together. */
- continue;
+ /* Simply link in duplicates and fix up the chain below. */
}
-
- /* If init_b == init_a + the size of the type * k, we have an
- interleaving, and DRA is accessed before DRB. */
- HOST_WIDE_INT type_size_a = tree_to_uhwi (sza);
- if (type_size_a == 0
- || (init_b - init_a) % type_size_a != 0)
- break;
-
- /* If we have a store, the accesses are adjacent. This splits
- groups into chunks we support (we don't support vectorization
- of stores with gaps). */
- if (!DR_IS_READ (dra) && init_b - init_prev != type_size_a)
- break;
-
- /* If the step (if not zero or non-constant) is greater than the
- difference between data-refs' inits this splits groups into
- suitable sizes. */
- if (tree_fits_shwi_p (DR_STEP (dra)))
+ else
{
- HOST_WIDE_INT step = tree_to_shwi (DR_STEP (dra));
- if (step != 0 && step <= (init_b - init_a))
+ /* If init_b == init_a + the size of the type * k, we have an
+ interleaving, and DRA is accessed before DRB. */
+ HOST_WIDE_INT type_size_a = tree_to_uhwi (sza);
+ if (type_size_a == 0
+ || (init_b - init_a) % type_size_a != 0)
+ break;
+
+ /* If we have a store, the accesses are adjacent. This splits
+ groups into chunks we support (we don't support vectorization
+ of stores with gaps). */
+ if (!DR_IS_READ (dra) && init_b - init_prev != type_size_a)
break;
+
+ /* If the step (if not zero or non-constant) is greater than the
+ difference between data-refs' inits this splits groups into
+ suitable sizes. */
+ if (tree_fits_shwi_p (DR_STEP (dra)))
+ {
+ HOST_WIDE_INT step = tree_to_shwi (DR_STEP (dra));
+ if (step != 0 && step <= (init_b - init_a))
+ break;
+ }
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "Detected interleaving ");
- if (DR_IS_READ (dra))
- dump_printf (MSG_NOTE, "load ");
- else
- dump_printf (MSG_NOTE, "store ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dra));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (drb));
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ DR_IS_READ (dra)
+ ? "Detected interleaving load %T and %T\n"
+ : "Detected interleaving store %T and %T\n",
+ DR_REF (dra), DR_REF (drb));
/* Link the found element into the group list. */
if (!DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
DR_GROUP_FIRST_ELEMENT (stmtinfo_b) = stmtinfo_a;
DR_GROUP_NEXT_ELEMENT (lastinfo) = stmtinfo_b;
lastinfo = stmtinfo_b;
+
+ STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a)
+ = !can_group_stmts_p (stmtinfo_a, stmtinfo_b, false);
+
+ if (dump_enabled_p () && STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Load suitable for SLP vectorization only.\n");
+
+ if (init_b == init_prev
+ && !to_fixup.add (DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
+ && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "Queuing group with duplicate access for fixup\n");
+ }
+ }
+
+ /* Fixup groups with duplicate entries by splitting it. */
+ while (1)
+ {
+ hash_set<stmt_vec_info>::iterator it = to_fixup.begin ();
+ if (!(it != to_fixup.end ()))
+ break;
+ stmt_vec_info grp = *it;
+ to_fixup.remove (grp);
+
+ /* Find the earliest duplicate group member. */
+ unsigned first_duplicate = -1u;
+ stmt_vec_info next, g = grp;
+ while ((next = DR_GROUP_NEXT_ELEMENT (g)))
+ {
+ if (tree_int_cst_equal (DR_INIT (STMT_VINFO_DR_INFO (next)->dr),
+ DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
+ && gimple_uid (STMT_VINFO_STMT (next)) < first_duplicate)
+ first_duplicate = gimple_uid (STMT_VINFO_STMT (next));
+ g = next;
+ }
+ if (first_duplicate == -1U)
+ continue;
+
+ /* Then move all stmts after the first duplicate to a new group.
+ Note this is a heuristic but one with the property that *it
+ is fixed up completely. */
+ g = grp;
+ stmt_vec_info newgroup = NULL, ng = grp;
+ while ((next = DR_GROUP_NEXT_ELEMENT (g)))
+ {
+ if (gimple_uid (STMT_VINFO_STMT (next)) >= first_duplicate)
+ {
+ DR_GROUP_NEXT_ELEMENT (g) = DR_GROUP_NEXT_ELEMENT (next);
+ if (!newgroup)
+ newgroup = next;
+ else
+ DR_GROUP_NEXT_ELEMENT (ng) = next;
+ ng = next;
+ DR_GROUP_FIRST_ELEMENT (ng) = newgroup;
+ }
+ else
+ g = DR_GROUP_NEXT_ELEMENT (g);
}
+ DR_GROUP_NEXT_ELEMENT (ng) = NULL;
+
+ /* Fixup the new group which still may contain duplicates. */
+ to_fixup.add (newgroup);
}
FOR_EACH_VEC_ELT (datarefs_copy, i, dr)
- if (STMT_VINFO_VECTORIZABLE (vect_dr_stmt (dr))
- && !vect_analyze_data_ref_access (dr))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: complicated access pattern.\n");
+ {
+ dr_vec_info *dr_info = vinfo->lookup_dr (dr);
+ if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
+ && !vect_analyze_data_ref_access (vinfo, dr_info))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not vectorized: complicated access pattern.\n");
- if (is_a <bb_vec_info> (vinfo))
- {
- /* Mark the statement as not vectorizable. */
- STMT_VINFO_VECTORIZABLE (vect_dr_stmt (dr)) = false;
- continue;
- }
- else
- {
- datarefs_copy.release ();
- return false;
- }
- }
+ if (is_a <bb_vec_info> (vinfo))
+ {
+ /* Mark the statement as not vectorizable. */
+ STMT_VINFO_VECTORIZABLE (dr_info->stmt) = false;
+ continue;
+ }
+ else
+ {
+ datarefs_copy.release ();
+ return opt_result::failure_at (dr_info->stmt->stmt,
+ "not vectorized:"
+ " complicated access pattern.\n");
+ }
+ }
+ }
datarefs_copy.release ();
- return true;
+ return opt_result::success ();
}
/* Function vect_vfa_segment_size.
Input:
- DR: The data reference.
+ DR_INFO: The data reference.
LENGTH_FACTOR: segment length to consider.
Return a value suitable for the dr_with_seg_len::seg_len field.
the size of the access; in effect it only describes the first byte. */
static tree
-vect_vfa_segment_size (struct data_reference *dr, tree length_factor)
+vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
{
length_factor = size_binop (MINUS_EXPR,
fold_convert (sizetype, length_factor),
size_one_node);
- return size_binop (MULT_EXPR, fold_convert (sizetype, DR_STEP (dr)),
+ return size_binop (MULT_EXPR, fold_convert (sizetype, DR_STEP (dr_info->dr)),
length_factor);
}
-/* Return a value that, when added to abs (vect_vfa_segment_size (dr)),
+/* Return a value that, when added to abs (vect_vfa_segment_size (DR_INFO)),
gives the worst-case number of bytes covered by the segment. */
static unsigned HOST_WIDE_INT
-vect_vfa_access_size (data_reference *dr)
+vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
{
- stmt_vec_info stmt_vinfo = vect_dr_stmt (dr);
- tree ref_type = TREE_TYPE (DR_REF (dr));
+ stmt_vec_info stmt_vinfo = dr_info->stmt;
+ tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
unsigned HOST_WIDE_INT ref_size = tree_to_uhwi (TYPE_SIZE_UNIT (ref_type));
unsigned HOST_WIDE_INT access_size = ref_size;
if (DR_GROUP_FIRST_ELEMENT (stmt_vinfo))
{
- gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == vect_dr_stmt (dr));
+ gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == stmt_vinfo);
access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
}
if (STMT_VINFO_VEC_STMT (stmt_vinfo)
- && (vect_supportable_dr_alignment (dr, false)
+ && (vect_supportable_dr_alignment (vinfo, dr_info, false)
== dr_explicit_realign_optimized))
{
/* We might access a full vector's worth. */
return access_size;
}
-/* Get the minimum alignment for all the scalar accesses that DR describes. */
+/* Get the minimum alignment for all the scalar accesses that DR_INFO
+ describes. */
static unsigned int
-vect_vfa_align (const data_reference *dr)
+vect_vfa_align (dr_vec_info *dr_info)
{
- return TYPE_ALIGN_UNIT (TREE_TYPE (DR_REF (dr)));
+ return TYPE_ALIGN_UNIT (TREE_TYPE (DR_REF (dr_info->dr)));
}
/* Function vect_no_alias_p.
of dr_with_seg_len::{seg_len,access_size} for A and B. */
static int
-vect_compile_time_alias (struct data_reference *a, struct data_reference *b,
+vect_compile_time_alias (dr_vec_info *a, dr_vec_info *b,
tree segment_length_a, tree segment_length_b,
unsigned HOST_WIDE_INT access_size_a,
unsigned HOST_WIDE_INT access_size_b)
{
- poly_offset_int offset_a = wi::to_poly_offset (DR_INIT (a));
- poly_offset_int offset_b = wi::to_poly_offset (DR_INIT (b));
+ poly_offset_int offset_a = wi::to_poly_offset (DR_INIT (a->dr));
+ poly_offset_int offset_b = wi::to_poly_offset (DR_INIT (b->dr));
poly_uint64 const_length_a;
poly_uint64 const_length_b;
/* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
bytes, e.g., int a[3] -> a[1] range is [a+4, a+16) instead of
[a, a+12) */
- if (tree_int_cst_compare (DR_STEP (a), size_zero_node) < 0)
+ if (tree_int_cst_compare (DR_STEP (a->dr), size_zero_node) < 0)
{
const_length_a = (-wi::to_poly_wide (segment_length_a)).force_uhwi ();
- offset_a = (offset_a + access_size_a) - const_length_a;
+ offset_a -= const_length_a;
}
else
const_length_a = tree_to_poly_uint64 (segment_length_a);
- if (tree_int_cst_compare (DR_STEP (b), size_zero_node) < 0)
+ if (tree_int_cst_compare (DR_STEP (b->dr), size_zero_node) < 0)
{
const_length_b = (-wi::to_poly_wide (segment_length_b)).force_uhwi ();
- offset_b = (offset_b + access_size_b) - const_length_b;
+ offset_b -= const_length_b;
}
else
const_length_b = tree_to_poly_uint64 (segment_length_b);
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "dependence distance between ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (DDR_A (ddr)));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (DDR_B (ddr)));
- dump_printf (MSG_NOTE, " is >= VF\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "dependence distance between %T and %T is >= VF\n",
+ DR_REF (DDR_A (ddr)), DR_REF (DDR_B (ddr)));
return true;
}
static void
dump_lower_bound (dump_flags_t dump_kind, const vec_lower_bound &lower_bound)
{
- dump_printf (dump_kind, "%s (", lower_bound.unsigned_p ? "unsigned" : "abs");
- dump_generic_expr (dump_kind, TDF_SLIM, lower_bound.expr);
- dump_printf (dump_kind, ") >= ");
+ dump_printf (dump_kind, "%s (%T) >= ",
+ lower_bound.unsigned_p ? "unsigned" : "abs",
+ lower_bound.expr);
dump_dec (dump_kind, lower_bound.min_value);
}
LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).safe_push (lower_bound);
}
-/* Return true if it's unlikely that the step of the vectorized form of DR
+/* Return true if it's unlikely that the step of the vectorized form of DR_INFO
will span fewer than GAP bytes. */
static bool
-vect_small_gap_p (loop_vec_info loop_vinfo, data_reference *dr, poly_int64 gap)
+vect_small_gap_p (loop_vec_info loop_vinfo, dr_vec_info *dr_info,
+ poly_int64 gap)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ stmt_vec_info stmt_info = dr_info->stmt;
HOST_WIDE_INT count
= estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
if (DR_GROUP_FIRST_ELEMENT (stmt_info))
count *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
- return estimated_poly_value (gap) <= count * vect_get_scalar_dr_size (dr);
+ return (estimated_poly_value (gap)
+ <= count * vect_get_scalar_dr_size (dr_info));
}
-/* Return true if we know that there is no alias between DR_A and DR_B
- when abs (DR_STEP (DR_A)) >= N for some N. When returning true, set
- *LOWER_BOUND_OUT to this N. */
+/* Return true if we know that there is no alias between DR_INFO_A and
+ DR_INFO_B when abs (DR_STEP (DR_INFO_A->dr)) >= N for some N.
+ When returning true, set *LOWER_BOUND_OUT to this N. */
static bool
-vectorizable_with_step_bound_p (data_reference *dr_a, data_reference *dr_b,
+vectorizable_with_step_bound_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b,
poly_uint64 *lower_bound_out)
{
/* Check that there is a constant gap of known sign between DR_A
and DR_B. */
+ data_reference *dr_a = dr_info_a->dr;
+ data_reference *dr_b = dr_info_b->dr;
poly_int64 init_a, init_b;
if (!operand_equal_p (DR_BASE_ADDRESS (dr_a), DR_BASE_ADDRESS (dr_b), 0)
|| !operand_equal_p (DR_OFFSET (dr_a), DR_OFFSET (dr_b), 0)
if (maybe_lt (init_b, init_a))
{
std::swap (init_a, init_b);
+ std::swap (dr_info_a, dr_info_b);
std::swap (dr_a, dr_b);
}
/* If the two accesses could be dependent within a scalar iteration,
make sure that we'd retain their order. */
- if (maybe_gt (init_a + vect_get_scalar_dr_size (dr_a), init_b)
- && !vect_preserves_scalar_order_p (vect_dr_stmt (dr_a),
- vect_dr_stmt (dr_b)))
+ if (maybe_gt (init_a + vect_get_scalar_dr_size (dr_info_a), init_b)
+ && !vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
return false;
/* There is no alias if abs (DR_STEP) is greater than or equal to
the bytes spanned by the combination of the two accesses. */
- *lower_bound_out = init_b + vect_get_scalar_dr_size (dr_b) - init_a;
+ *lower_bound_out = init_b + vect_get_scalar_dr_size (dr_info_b) - init_a;
return true;
}
Return FALSE if resulting list of ddrs is longer then allowed by
PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE. */
-bool
+opt_result
vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
{
typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
}
if (may_alias_ddrs.is_empty ())
- return true;
+ return opt_result::success ();
comp_alias_ddrs.create (may_alias_ddrs.length ());
/* First, we collect all data ref pairs for aliasing checks. */
FOR_EACH_VEC_ELT (may_alias_ddrs, i, ddr)
{
- int comp_res;
poly_uint64 lower_bound;
- struct data_reference *dr_a, *dr_b;
tree segment_length_a, segment_length_b;
unsigned HOST_WIDE_INT access_size_a, access_size_b;
unsigned int align_a, align_b;
if (!compared_objects.add (new_pair))
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "checking that ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, new_pair.first);
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, new_pair.second);
- dump_printf (MSG_NOTE, " have different addresses\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "checking that %T and %T"
+ " have different addresses\n",
+ new_pair.first, new_pair.second);
LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).safe_push (new_pair);
}
continue;
}
- dr_a = DDR_A (ddr);
- stmt_vec_info stmt_info_a = vect_dr_stmt (DDR_A (ddr));
+ dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
+ stmt_vec_info stmt_info_a = dr_info_a->stmt;
- dr_b = DDR_B (ddr);
- stmt_vec_info stmt_info_b = vect_dr_stmt (DDR_B (ddr));
+ dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
+ stmt_vec_info stmt_info_b = dr_info_b->stmt;
+
+ bool preserves_scalar_order_p
+ = vect_preserves_scalar_order_p (dr_info_a, dr_info_b);
/* Skip the pair if inter-iteration dependencies are irrelevant
and intra-iteration dependencies are guaranteed to be honored. */
if (ignore_step_p
- && (vect_preserves_scalar_order_p (stmt_info_a, stmt_info_b)
- || vectorizable_with_step_bound_p (dr_a, dr_b, &lower_bound)))
+ && (preserves_scalar_order_p
+ || vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
+ &lower_bound)))
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "no need for alias check between ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_a));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_b));
- dump_printf (MSG_NOTE, " when VF is 1\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "no need for alias check between "
+ "%T and %T when VF is 1\n",
+ DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
continue;
}
(It might not be, for example, if the minimum step is much larger
than the number of bytes handled by one vector iteration.) */
if (!ignore_step_p
- && TREE_CODE (DR_STEP (dr_a)) != INTEGER_CST
- && vectorizable_with_step_bound_p (dr_a, dr_b, &lower_bound)
- && (vect_small_gap_p (loop_vinfo, dr_a, lower_bound)
- || vect_small_gap_p (loop_vinfo, dr_b, lower_bound)))
+ && TREE_CODE (DR_STEP (dr_info_a->dr)) != INTEGER_CST
+ && vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
+ &lower_bound)
+ && (vect_small_gap_p (loop_vinfo, dr_info_a, lower_bound)
+ || vect_small_gap_p (loop_vinfo, dr_info_b, lower_bound)))
{
- bool unsigned_p = dr_known_forward_stride_p (dr_a);
+ bool unsigned_p = dr_known_forward_stride_p (dr_info_a->dr);
if (dump_enabled_p ())
{
- dump_printf_loc (MSG_NOTE, vect_location, "no alias between ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_a));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_b));
- dump_printf (MSG_NOTE, " when the step ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_STEP (dr_a));
- dump_printf (MSG_NOTE, " is outside ");
+ dump_printf_loc (MSG_NOTE, vect_location, "no alias between "
+ "%T and %T when the step %T is outside ",
+ DR_REF (dr_info_a->dr),
+ DR_REF (dr_info_b->dr),
+ DR_STEP (dr_info_a->dr));
if (unsigned_p)
dump_printf (MSG_NOTE, "[0");
else
dump_dec (MSG_NOTE, lower_bound);
dump_printf (MSG_NOTE, ")\n");
}
- vect_check_lower_bound (loop_vinfo, DR_STEP (dr_a), unsigned_p,
- lower_bound);
+ vect_check_lower_bound (loop_vinfo, DR_STEP (dr_info_a->dr),
+ unsigned_p, lower_bound);
continue;
}
if (dr_group_first_a)
{
stmt_info_a = dr_group_first_a;
- dr_a = STMT_VINFO_DATA_REF (stmt_info_a);
+ dr_info_a = STMT_VINFO_DR_INFO (stmt_info_a);
}
stmt_vec_info dr_group_first_b = DR_GROUP_FIRST_ELEMENT (stmt_info_b);
if (dr_group_first_b)
{
stmt_info_b = dr_group_first_b;
- dr_b = STMT_VINFO_DATA_REF (stmt_info_b);
+ dr_info_b = STMT_VINFO_DR_INFO (stmt_info_b);
}
if (ignore_step_p)
}
else
{
- if (!operand_equal_p (DR_STEP (dr_a), DR_STEP (dr_b), 0))
+ if (!operand_equal_p (DR_STEP (dr_info_a->dr),
+ DR_STEP (dr_info_b->dr), 0))
length_factor = scalar_loop_iters;
else
length_factor = size_int (vect_factor);
- segment_length_a = vect_vfa_segment_size (dr_a, length_factor);
- segment_length_b = vect_vfa_segment_size (dr_b, length_factor);
+ segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
+ segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
}
- access_size_a = vect_vfa_access_size (dr_a);
- access_size_b = vect_vfa_access_size (dr_b);
- align_a = vect_vfa_align (dr_a);
- align_b = vect_vfa_align (dr_b);
-
- comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (dr_a),
- DR_BASE_ADDRESS (dr_b));
- if (comp_res == 0)
- comp_res = data_ref_compare_tree (DR_OFFSET (dr_a),
- DR_OFFSET (dr_b));
+ access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
+ access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
+ align_a = vect_vfa_align (dr_info_a);
+ align_b = vect_vfa_align (dr_info_b);
/* See whether the alias is known at compilation time. */
- if (comp_res == 0
- && TREE_CODE (DR_STEP (dr_a)) == INTEGER_CST
- && TREE_CODE (DR_STEP (dr_b)) == INTEGER_CST
+ if (operand_equal_p (DR_BASE_ADDRESS (dr_info_a->dr),
+ DR_BASE_ADDRESS (dr_info_b->dr), 0)
+ && operand_equal_p (DR_OFFSET (dr_info_a->dr),
+ DR_OFFSET (dr_info_b->dr), 0)
+ && TREE_CODE (DR_STEP (dr_info_a->dr)) == INTEGER_CST
+ && TREE_CODE (DR_STEP (dr_info_b->dr)) == INTEGER_CST
&& poly_int_tree_p (segment_length_a)
&& poly_int_tree_p (segment_length_b))
{
- int res = vect_compile_time_alias (dr_a, dr_b,
+ int res = vect_compile_time_alias (dr_info_a, dr_info_b,
segment_length_a,
segment_length_b,
access_size_a,
if (res >= 0 && dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
- "can tell at compile time that ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_a));
- dump_printf (MSG_NOTE, " and ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_REF (dr_b));
+ "can tell at compile time that %T and %T",
+ DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
if (res == 0)
dump_printf (MSG_NOTE, " do not alias\n");
else
continue;
if (res == 1)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "not vectorized: compilation time alias.\n");
- return false;
- }
+ return opt_result::failure_at (stmt_info_b->stmt,
+ "not vectorized:"
+ " compilation time alias: %G%G",
+ stmt_info_a->stmt,
+ stmt_info_b->stmt);
}
- dr_with_seg_len_pair_t dr_with_seg_len_pair
- (dr_with_seg_len (dr_a, segment_length_a, access_size_a, align_a),
- dr_with_seg_len (dr_b, segment_length_b, access_size_b, align_b));
+ dr_with_seg_len dr_a (dr_info_a->dr, segment_length_a,
+ access_size_a, align_a);
+ dr_with_seg_len dr_b (dr_info_b->dr, segment_length_b,
+ access_size_b, align_b);
+ /* Canonicalize the order to be the one that's needed for accurate
+ RAW, WAR and WAW flags, in cases where the data references are
+ well-ordered. The order doesn't really matter otherwise,
+ but we might as well be consistent. */
+ if (get_later_stmt (stmt_info_a, stmt_info_b) == stmt_info_a)
+ std::swap (dr_a, dr_b);
- /* Canonicalize pairs by sorting the two DR members. */
- if (comp_res > 0)
- std::swap (dr_with_seg_len_pair.first, dr_with_seg_len_pair.second);
+ dr_with_seg_len_pair_t dr_with_seg_len_pair
+ (dr_a, dr_b, (preserves_scalar_order_p
+ ? dr_with_seg_len_pair_t::WELL_ORDERED
+ : dr_with_seg_len_pair_t::REORDERED));
comp_alias_ddrs.safe_push (dr_with_seg_len_pair);
}
unsigned int count = (comp_alias_ddrs.length ()
+ check_unequal_addrs.length ());
- dump_printf_loc (MSG_NOTE, vect_location,
- "improved number of alias checks from %d to %d\n",
- may_alias_ddrs.length (), count);
- if ((int) count > PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "number of versioning for alias "
- "run-time tests exceeds %d "
- "(--param vect-max-version-for-alias-checks)\n",
- PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS));
- return false;
- }
-
- return true;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "improved number of alias checks from %d to %d\n",
+ may_alias_ddrs.length (), count);
+ unsigned limit = param_vect_max_version_for_alias_checks;
+ if (flag_simd_cost_model == VECT_COST_MODEL_CHEAP)
+ limit = param_vect_max_version_for_alias_checks * 6 / 10;
+ if (count > limit)
+ return opt_result::failure_at
+ (vect_location,
+ "number of versioning for alias run-time tests exceeds %d "
+ "(--param vect-max-version-for-alias-checks)\n", limit);
+
+ return opt_result::success ();
}
/* Check whether we can use an internal function for a gather load
or scatter store. READ_P is true for loads and false for stores.
MASKED_P is true if the load or store is conditional. MEMORY_TYPE is
- the type of the memory elements being loaded or stored. OFFSET_BITS
- is the number of bits in each scalar offset and OFFSET_SIGN is the
- sign of the offset. SCALE is the amount by which the offset should
+ the type of the memory elements being loaded or stored. OFFSET_TYPE
+ is the type of the offset that is being applied to the invariant
+ base address. SCALE is the amount by which the offset should
be multiplied *after* it has been converted to address width.
- Return true if the function is supported, storing the function
- id in *IFN_OUT and the type of a vector element in *ELEMENT_TYPE_OUT. */
+ Return true if the function is supported, storing the function id in
+ *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT. */
bool
-vect_gather_scatter_fn_p (bool read_p, bool masked_p, tree vectype,
- tree memory_type, unsigned int offset_bits,
- signop offset_sign, int scale,
- internal_fn *ifn_out, tree *element_type_out)
+vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
+ tree vectype, tree memory_type, tree offset_type,
+ int scale, internal_fn *ifn_out,
+ tree *offset_vectype_out)
{
unsigned int memory_bits = tree_to_uhwi (TYPE_SIZE (memory_type));
- unsigned int element_bits = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
- if (offset_bits > element_bits)
- /* Internal functions require the offset to be the same width as
- the vector elements. We can extend narrower offsets, but it isn't
- safe to truncate wider offsets. */
- return false;
-
+ unsigned int element_bits = vector_element_bits (vectype);
if (element_bits != memory_bits)
/* For now the vector elements must be the same width as the
memory elements. */
else
ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
- /* Test whether the target supports this combination. */
- if (!internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
- offset_sign, scale))
- return false;
+ for (;;)
+ {
+ tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
+ if (!offset_vectype)
+ return false;
- *ifn_out = ifn;
- *element_type_out = TREE_TYPE (vectype);
- return true;
+ /* Test whether the target supports this combination. */
+ if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
+ offset_vectype, scale))
+ {
+ *ifn_out = ifn;
+ *offset_vectype_out = offset_vectype;
+ return true;
+ }
+
+ if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
+ && TYPE_PRECISION (offset_type) >= element_bits)
+ return false;
+
+ offset_type = build_nonstandard_integer_type
+ (TYPE_PRECISION (offset_type) * 2, TYPE_UNSIGNED (offset_type));
+ }
}
/* STMT_INFO is a call to an internal gather load or scatter store function.
{
HOST_WIDE_INT scale = 1;
poly_int64 pbitpos, pbitsize;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree offtype = NULL_TREE;
tree decl = NULL_TREE, base, off;
machine_mode pmode;
int punsignedp, reversep, pvolatilep = 0;
internal_fn ifn;
- tree element_type;
+ tree offset_vectype;
bool masked_p = false;
/* See whether this is already a call to a gather/scatter internal function.
{
int new_scale = tree_to_shwi (op1);
/* Only treat this as a scaling operation if the target
- supports it. */
+ supports it for at least some offset type. */
if (use_ifn_p
- && !vect_gather_scatter_fn_p (DR_IS_READ (dr), masked_p,
- vectype, memory_type, 1,
- TYPE_SIGN (TREE_TYPE (op0)),
+ && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
+ masked_p, vectype, memory_type,
+ signed_char_type_node,
+ new_scale, &ifn,
+ &offset_vectype)
+ && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
+ masked_p, vectype, memory_type,
+ unsigned_char_type_node,
new_scale, &ifn,
- &element_type))
+ &offset_vectype))
break;
scale = new_scale;
off = op0;
if (!POINTER_TYPE_P (TREE_TYPE (op0))
&& !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
break;
+
+ /* Don't include the conversion if the target is happy with
+ the current offset type. */
+ if (use_ifn_p
+ && vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
+ masked_p, vectype, memory_type,
+ TREE_TYPE (off), scale, &ifn,
+ &offset_vectype))
+ break;
+
if (TYPE_PRECISION (TREE_TYPE (op0))
== TYPE_PRECISION (TREE_TYPE (off)))
{
continue;
}
- /* The internal functions need the offset to be the same width
- as the elements of VECTYPE. Don't include operations that
- cast the offset from that width to a different width. */
- if (use_ifn_p
- && (int_size_in_bytes (TREE_TYPE (vectype))
- == int_size_in_bytes (TREE_TYPE (off))))
- break;
-
if (TYPE_PRECISION (TREE_TYPE (op0))
< TYPE_PRECISION (TREE_TYPE (off)))
{
if (use_ifn_p)
{
- if (!vect_gather_scatter_fn_p (DR_IS_READ (dr), masked_p, vectype,
- memory_type, TYPE_PRECISION (offtype),
- TYPE_SIGN (offtype), scale, &ifn,
- &element_type))
+ if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
+ vectype, memory_type, offtype, scale,
+ &ifn, &offset_vectype))
return false;
}
else
return false;
ifn = IFN_LAST;
- element_type = TREE_TYPE (vectype);
+ /* The offset vector type will be read from DECL when needed. */
+ offset_vectype = NULL_TREE;
}
info->ifn = ifn;
info->base = base;
info->offset = off;
info->offset_dt = vect_unknown_def_type;
- info->offset_vectype = NULL_TREE;
+ info->offset_vectype = offset_vectype;
info->scale = scale;
- info->element_type = element_type;
+ info->element_type = TREE_TYPE (vectype);
info->memory_type = memory_type;
return true;
}
append them to DATAREFS. Return false if datarefs in this stmt cannot
be handled. */
-bool
+opt_result
vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
vec<data_reference_p> *datarefs)
{
loop vectorization and BB vectorization checks dependences with a
stmt walk. */
if (gimple_clobber_p (stmt))
- return true;
+ return opt_result::success ();
if (gimple_has_volatile_ops (stmt))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: volatile type ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
- }
- return false;
- }
+ return opt_result::failure_at (stmt, "not vectorized: volatile type: %G",
+ stmt);
- if (stmt_can_throw_internal (stmt))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: statement can throw an "
- "exception ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
- }
- return false;
- }
+ if (stmt_can_throw_internal (cfun, stmt))
+ return opt_result::failure_at (stmt,
+ "not vectorized:"
+ " statement can throw an exception: %G",
+ stmt);
auto_vec<data_reference_p, 2> refs;
- if (!find_data_references_in_stmt (loop, stmt, &refs))
- return false;
+ opt_result res = find_data_references_in_stmt (loop, stmt, &refs);
+ if (!res)
+ return res;
if (refs.is_empty ())
- return true;
+ return opt_result::success ();
if (refs.length () > 1)
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: more than one data ref "
- "in stmt: ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
- }
- return false;
- }
+ return opt_result::failure_at (stmt,
+ "not vectorized:"
+ " more than one data ref in stmt: %G", stmt);
if (gcall *call = dyn_cast <gcall *> (stmt))
if (!gimple_call_internal_p (call)
|| (gimple_call_internal_fn (call) != IFN_MASK_LOAD
&& gimple_call_internal_fn (call) != IFN_MASK_STORE))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: dr in a call ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
- }
- return false;
- }
+ return opt_result::failure_at (stmt,
+ "not vectorized: dr in a call %G", stmt);
data_reference_p dr = refs.pop ();
if (TREE_CODE (DR_REF (dr)) == COMPONENT_REF
&& DECL_BIT_FIELD (TREE_OPERAND (DR_REF (dr), 1)))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: statement is bitfield "
- "access ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, stmt, 0);
- }
- return false;
- }
+ return opt_result::failure_at (stmt,
+ "not vectorized:"
+ " statement is bitfield access %G", stmt);
if (DR_BASE_ADDRESS (dr)
&& TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: base addr of dr is a "
- "constant\n");
- return false;
- }
+ return opt_result::failure_at (stmt,
+ "not vectorized:"
+ " base addr of dr is a constant\n");
/* Check whether this may be a SIMD lane access and adjust the
DR to make it easier for us to handle it. */
&& DR_OFFSET (newdr)
&& DR_INIT (newdr)
&& DR_STEP (newdr)
+ && TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
&& integer_zerop (DR_STEP (newdr)))
{
+ tree base_address = DR_BASE_ADDRESS (newdr);
tree off = DR_OFFSET (newdr);
+ tree step = ssize_int (1);
+ if (integer_zerop (off)
+ && TREE_CODE (base_address) == POINTER_PLUS_EXPR)
+ {
+ off = TREE_OPERAND (base_address, 1);
+ base_address = TREE_OPERAND (base_address, 0);
+ }
STRIP_NOPS (off);
- if (TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
- && TREE_CODE (off) == MULT_EXPR
+ if (TREE_CODE (off) == MULT_EXPR
&& tree_fits_uhwi_p (TREE_OPERAND (off, 1)))
{
- tree step = TREE_OPERAND (off, 1);
+ step = TREE_OPERAND (off, 1);
off = TREE_OPERAND (off, 0);
STRIP_NOPS (off);
- if (CONVERT_EXPR_P (off)
- && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
- < TYPE_PRECISION (TREE_TYPE (off))))
- off = TREE_OPERAND (off, 0);
- if (TREE_CODE (off) == SSA_NAME)
+ }
+ if (CONVERT_EXPR_P (off)
+ && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
+ < TYPE_PRECISION (TREE_TYPE (off))))
+ off = TREE_OPERAND (off, 0);
+ if (TREE_CODE (off) == SSA_NAME)
+ {
+ gimple *def = SSA_NAME_DEF_STMT (off);
+ /* Look through widening conversion. */
+ if (is_gimple_assign (def)
+ && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
{
- gimple *def = SSA_NAME_DEF_STMT (off);
+ tree rhs1 = gimple_assign_rhs1 (def);
+ if (TREE_CODE (rhs1) == SSA_NAME
+ && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
+ && (TYPE_PRECISION (TREE_TYPE (off))
+ > TYPE_PRECISION (TREE_TYPE (rhs1))))
+ def = SSA_NAME_DEF_STMT (rhs1);
+ }
+ if (is_gimple_call (def)
+ && gimple_call_internal_p (def)
+ && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
+ {
+ tree arg = gimple_call_arg (def, 0);
tree reft = TREE_TYPE (DR_REF (newdr));
- if (is_gimple_call (def)
- && gimple_call_internal_p (def)
- && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
+ gcc_assert (TREE_CODE (arg) == SSA_NAME);
+ arg = SSA_NAME_VAR (arg);
+ if (arg == loop->simduid
+ /* For now. */
+ && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
{
- tree arg = gimple_call_arg (def, 0);
- gcc_assert (TREE_CODE (arg) == SSA_NAME);
- arg = SSA_NAME_VAR (arg);
- if (arg == loop->simduid
- /* For now. */
- && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
- {
- DR_OFFSET (newdr) = ssize_int (0);
- DR_STEP (newdr) = step;
- DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
- DR_STEP_ALIGNMENT (newdr)
- = highest_pow2_factor (step);
- /* Mark as simd-lane access. */
- newdr->aux = (void *)-1;
- free_data_ref (dr);
- datarefs->safe_push (newdr);
- return true;
- }
+ DR_BASE_ADDRESS (newdr) = base_address;
+ DR_OFFSET (newdr) = ssize_int (0);
+ DR_STEP (newdr) = step;
+ DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
+ DR_STEP_ALIGNMENT (newdr) = highest_pow2_factor (step);
+ /* Mark as simd-lane access. */
+ tree arg2 = gimple_call_arg (def, 1);
+ newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
+ free_data_ref (dr);
+ datarefs->safe_push (newdr);
+ return opt_result::success ();
}
}
}
}
datarefs->safe_push (dr);
- return true;
+ return opt_result::success ();
}
/* Function vect_analyze_data_refs.
*/
-bool
-vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf)
+opt_result
+vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
{
- struct loop *loop = NULL;
+ class loop *loop = NULL;
unsigned int i;
struct data_reference *dr;
tree scalar_type;
if (gatherscatter == SG_NONE)
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: data ref analysis "
- "failed ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not vectorized: data ref analysis "
+ "failed %G", stmt_info->stmt);
if (is_a <bb_vec_info> (vinfo))
{
/* In BB vectorization the ref can still participate
STMT_VINFO_VECTORIZABLE (stmt_info) = false;
continue;
}
- return false;
+ return opt_result::failure_at (stmt_info->stmt,
+ "not vectorized:"
+ " data ref analysis failed: %G",
+ stmt_info->stmt);
}
}
/* See if this was detected as SIMD lane access. */
- if (dr->aux == (void *)-1)
+ if (dr->aux == (void *)-1
+ || dr->aux == (void *)-2
+ || dr->aux == (void *)-3
+ || dr->aux == (void *)-4)
{
if (nested_in_vect_loop_p (loop, stmt_info))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: data ref analysis "
- "failed ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
- return false;
- }
- STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) = true;
+ return opt_result::failure_at (stmt_info->stmt,
+ "not vectorized:"
+ " data ref analysis failed: %G",
+ stmt_info->stmt);
+ STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
+ = -(uintptr_t) dr->aux;
}
tree base = get_base_address (DR_REF (dr));
if (base && VAR_P (base) && DECL_NONALIASED (base))
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: base object not addressable "
- "for stmt: ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not vectorized: base object not addressable "
+ "for stmt: %G", stmt_info->stmt);
if (is_a <bb_vec_info> (vinfo))
{
/* In BB vectorization the ref can still participate
STMT_VINFO_VECTORIZABLE (stmt_info) = false;
continue;
}
- return false;
+ return opt_result::failure_at (stmt_info->stmt,
+ "not vectorized: base object not"
+ " addressable for stmt: %G",
+ stmt_info->stmt);
}
if (is_a <loop_vec_info> (vinfo)
&& TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
{
if (nested_in_vect_loop_p (loop, stmt_info))
- {
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: not suitable for strided "
- "load ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
- return false;
- }
+ return opt_result::failure_at (stmt_info->stmt,
+ "not vectorized: "
+ "not suitable for strided load %G",
+ stmt_info->stmt);
STMT_VINFO_STRIDED_P (stmt_info) = true;
}
tree init_ref = build_fold_indirect_ref (init_addr);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "analyze in outer loop: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, init_ref);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "analyze in outer loop: %T\n", init_ref);
- if (!dr_analyze_innermost (&STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info),
- init_ref, loop))
+ opt_result res
+ = dr_analyze_innermost (&STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info),
+ init_ref, loop, stmt_info->stmt);
+ if (!res)
/* dr_analyze_innermost already explained the failure. */
- return false;
+ return res;
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "\touter base_address: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM,
- STMT_VINFO_DR_BASE_ADDRESS (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter offset from base address: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM,
- STMT_VINFO_DR_OFFSET (stmt_info));
- dump_printf (MSG_NOTE,
- "\n\touter constant offset from base address: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM,
- STMT_VINFO_DR_INIT (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter step: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM,
- STMT_VINFO_DR_STEP (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter base alignment: %d\n",
- STMT_VINFO_DR_BASE_ALIGNMENT (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter base misalignment: %d\n",
- STMT_VINFO_DR_BASE_MISALIGNMENT (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter offset alignment: %d\n",
- STMT_VINFO_DR_OFFSET_ALIGNMENT (stmt_info));
- dump_printf (MSG_NOTE, "\n\touter step alignment: %d\n",
- STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "\touter base_address: %T\n"
+ "\touter offset from base address: %T\n"
+ "\touter constant offset from base address: %T\n"
+ "\touter step: %T\n"
+ "\touter base alignment: %d\n\n"
+ "\touter base misalignment: %d\n"
+ "\touter offset alignment: %d\n"
+ "\touter step alignment: %d\n",
+ STMT_VINFO_DR_BASE_ADDRESS (stmt_info),
+ STMT_VINFO_DR_OFFSET (stmt_info),
+ STMT_VINFO_DR_INIT (stmt_info),
+ STMT_VINFO_DR_STEP (stmt_info),
+ STMT_VINFO_DR_BASE_ALIGNMENT (stmt_info),
+ STMT_VINFO_DR_BASE_MISALIGNMENT (stmt_info),
+ STMT_VINFO_DR_OFFSET_ALIGNMENT (stmt_info),
+ STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
}
/* Set vectype for STMT. */
scalar_type = TREE_TYPE (DR_REF (dr));
- STMT_VINFO_VECTYPE (stmt_info)
- = get_vectype_for_scalar_type (scalar_type);
- if (!STMT_VINFO_VECTYPE (stmt_info))
+ tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
+ if (!vectype)
{
if (dump_enabled_p ())
{
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: no vectype for stmt: ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
+ "not vectorized: no vectype for stmt: %G",
+ stmt_info->stmt);
dump_printf (MSG_MISSED_OPTIMIZATION, " scalar_type: ");
dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_DETAILS,
scalar_type);
STMT_VINFO_VECTORIZABLE (stmt_info) = false;
continue;
}
- return false;
+ if (fatal)
+ *fatal = false;
+ return opt_result::failure_at (stmt_info->stmt,
+ "not vectorized:"
+ " no vectype for stmt: %G"
+ " scalar_type: %T\n",
+ stmt_info->stmt, scalar_type);
}
else
{
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "got vectype for stmt: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt_info->stmt, 0);
- dump_generic_expr (MSG_NOTE, TDF_SLIM,
- STMT_VINFO_VECTYPE (stmt_info));
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "got vectype for stmt: %G%T\n",
+ stmt_info->stmt, vectype);
}
/* Adjust the minimal vectorization factor according to the
vector type. */
- vf = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
+ vf = TYPE_VECTOR_SUBPARTS (vectype);
*min_vf = upper_bound (*min_vf, vf);
+ /* Leave the BB vectorizer to pick the vector type later, based on
+ the final dataref group size and SLP node size. */
+ if (is_a <loop_vec_info> (vinfo))
+ STMT_VINFO_VECTYPE (stmt_info) = vectype;
+
if (gatherscatter != SG_NONE)
{
gather_scatter_info gs_info;
if (!vect_check_gather_scatter (stmt_info,
as_a <loop_vec_info> (vinfo),
&gs_info)
- || !get_vectype_for_scalar_type (TREE_TYPE (gs_info.offset)))
+ || !get_vectype_for_scalar_type (vinfo,
+ TREE_TYPE (gs_info.offset)))
{
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- (gatherscatter == GATHER) ?
- "not vectorized: not suitable for gather "
- "load " :
- "not vectorized: not suitable for scatter "
- "store ");
- dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM,
- stmt_info->stmt, 0);
- }
- return false;
+ if (fatal)
+ *fatal = false;
+ return opt_result::failure_at
+ (stmt_info->stmt,
+ (gatherscatter == GATHER)
+ ? "not vectorized: not suitable for gather load %G"
+ : "not vectorized: not suitable for scatter store %G",
+ stmt_info->stmt);
}
STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
}
longer need to. */
gcc_assert (i == datarefs.length ());
- return true;
+ return opt_result::success ();
}
return new_vect_var;
}
-/* Duplicate ptr info and set alignment/misaligment on NAME from DR. */
+/* Duplicate ptr info and set alignment/misaligment on NAME from DR_INFO. */
static void
-vect_duplicate_ssa_name_ptr_info (tree name, data_reference *dr)
+vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
{
- duplicate_ssa_name_ptr_info (name, DR_PTR_INFO (dr));
- int misalign = DR_MISALIGNMENT (dr);
+ duplicate_ssa_name_ptr_info (name, DR_PTR_INFO (dr_info->dr));
+ int misalign = DR_MISALIGNMENT (dr_info);
if (misalign == DR_MISALIGNMENT_UNKNOWN)
mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (name));
else
set_ptr_info_alignment (SSA_NAME_PTR_INFO (name),
- DR_TARGET_ALIGNMENT (dr), misalign);
+ known_alignment (DR_TARGET_ALIGNMENT (dr_info)),
+ misalign);
}
/* Function vect_create_addr_base_for_vector_ref.
FORNOW: We are only handling array accesses with step 1. */
tree
-vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info,
+vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_seq *new_stmt_list,
tree offset,
tree byte_offset)
{
- struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ struct data_reference *dr = dr_info->dr;
const char *base_name;
tree addr_base;
tree dest;
gimple_seq seq = NULL;
tree vect_ptr_type;
tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- innermost_loop_behavior *drb = vect_dr_behavior (dr);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
- tree base_offset = unshare_expr (drb->offset);
+ tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
&& TREE_CODE (addr_base) == SSA_NAME
&& !SSA_NAME_PTR_INFO (addr_base))
{
- vect_duplicate_ssa_name_ptr_info (addr_base, dr);
+ vect_duplicate_ssa_name_ptr_info (addr_base, dr_info);
if (offset || byte_offset)
mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (addr_base));
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "created ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, addr_base);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "created %T\n", addr_base);
return addr_base;
}
Return the increment stmt that updates the pointer in PTR_INCR.
- 3. Set INV_P to true if the access pattern of the data reference in the
- vectorized loop is invariant. Set it to false otherwise.
-
- 4. Return the pointer. */
+ 3. Return the pointer. */
tree
-vect_create_data_ref_ptr (stmt_vec_info stmt_info, tree aggr_type,
- struct loop *at_loop, tree offset,
+vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
+ tree aggr_type, class loop *at_loop, tree offset,
tree *initial_address, gimple_stmt_iterator *gsi,
- gimple **ptr_incr, bool only_init, bool *inv_p,
+ gimple **ptr_incr, bool only_init,
tree byte_offset, tree iv_step)
{
const char *base_name;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = NULL;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ class loop *loop = NULL;
bool nested_in_vect_loop = false;
- struct loop *containing_loop = NULL;
+ class loop *containing_loop = NULL;
tree aggr_ptr_type;
tree aggr_ptr;
tree new_temp;
edge pe = NULL;
basic_block new_bb;
tree aggr_ptr_init;
- struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ struct data_reference *dr = dr_info->dr;
tree aptr;
gimple_stmt_iterator incr_gsi;
bool insert_after;
tree indx_before_incr, indx_after_incr;
gimple *incr;
- tree step;
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
gcc_assert (iv_step != NULL_TREE
|| TREE_CODE (aggr_type) == ARRAY_TYPE
*ptr_incr = NULL;
}
- /* Check the step (evolution) of the load in LOOP, and record
- whether it's invariant. */
- step = vect_dr_behavior (dr)->step;
- if (integer_zerop (step))
- *inv_p = true;
- else
- *inv_p = false;
-
/* Create an expression for the first address accessed by this load
in LOOP. */
base_name = get_name (DR_BASE_ADDRESS (dr));
{
tree dr_base_type = TREE_TYPE (DR_BASE_OBJECT (dr));
dump_printf_loc (MSG_NOTE, vect_location,
- "create %s-pointer variable to type: ",
- get_tree_code_name (TREE_CODE (aggr_type)));
- dump_generic_expr (MSG_NOTE, TDF_SLIM, aggr_type);
+ "create %s-pointer variable to type: %T",
+ get_tree_code_name (TREE_CODE (aggr_type)),
+ aggr_type);
if (TREE_CODE (dr_base_type) == ARRAY_TYPE)
dump_printf (MSG_NOTE, " vectorizing an array ref: ");
else if (TREE_CODE (dr_base_type) == VECTOR_TYPE)
dump_printf (MSG_NOTE, " vectorizing a record based array ref: ");
else
dump_printf (MSG_NOTE, " vectorizing a pointer ref: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, DR_BASE_OBJECT (dr));
- dump_printf (MSG_NOTE, "\n");
+ dump_printf (MSG_NOTE, "%T\n", DR_BASE_OBJECT (dr));
}
/* (1) Create the new aggregate-pointer variable.
/* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */
- new_temp = vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
+ new_temp = vect_create_addr_base_for_vector_ref (vinfo,
+ stmt_info, &new_stmt_list,
offset, byte_offset);
if (new_stmt_list)
{
aptr = aggr_ptr_init;
else
{
+ /* Accesses to invariant addresses should be handled specially
+ by the caller. */
+ tree step = vect_dr_behavior (vinfo, dr_info)->step;
+ gcc_assert (!integer_zerop (step));
+
if (iv_step == NULL_TREE)
{
- /* The step of the aggregate pointer is the type size. */
+ /* The step of the aggregate pointer is the type size,
+ negated for downward accesses. */
iv_step = TYPE_SIZE_UNIT (aggr_type);
- /* One exception to the above is when the scalar step of the load in
- LOOP is zero. In this case the step here is also zero. */
- if (*inv_p)
- iv_step = size_zero_node;
- else if (tree_int_cst_sgn (step) == -1)
+ if (tree_int_cst_sgn (step) == -1)
iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
}
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
{
- vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr);
- vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr);
+ vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
+ vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
}
if (ptr_incr)
*ptr_incr = incr;
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
{
- vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr);
- vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr);
+ vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
+ vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
}
if (ptr_incr)
*ptr_incr = incr;
*/
tree
-bump_vector_ptr (tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
+bump_vector_ptr (vec_info *vinfo,
+ tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
stmt_vec_info stmt_info, tree bump)
{
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
dataref_ptr, update);
- vect_finish_stmt_generation (stmt_info, incr_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
if (dump_enabled_p ())
dump_printf (MSG_MISSED_OPTIMIZATION,
- "permutaion op not supported by target.\n");
+ "permutation op not supported by target.\n");
return false;
}
I4: 6 14 22 30 7 15 23 31. */
void
-vect_permute_store_chain (vec<tree> dr_chain,
+vect_permute_store_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect1 = data_ref;
vect2 = dr_chain[2];
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
vect2, perm3_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j] = data_ref;
}
}
high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
vect2, perm_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j] = high;
/* Create interleaving stmt:
low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
vect2, perm_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[2*j+1] = low;
}
memcpy (dr_chain.address (), result_chain->address (),
Return value - the result of the loop-header phi node. */
tree
-vect_setup_realignment (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
- tree *realignment_token,
+vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info,
+ gimple_stmt_iterator *gsi, tree *realignment_token,
enum dr_alignment_support alignment_support_scheme,
tree init_addr,
- struct loop **at_loop)
+ class loop **at_loop)
{
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
- struct loop *loop = NULL;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
+ struct data_reference *dr = dr_info->dr;
+ class loop *loop = NULL;
edge pe = NULL;
tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
tree vec_dest;
gphi *phi_stmt;
tree msq = NULL_TREE;
gimple_seq stmts = NULL;
- bool inv_p;
bool compute_in_loop = false;
bool nested_in_vect_loop = false;
- struct loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
- struct loop *loop_for_initial_load = NULL;
+ class loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
+ class loop *loop_for_initial_load = NULL;
if (loop_vinfo)
{
gcc_assert (!compute_in_loop);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- ptr = vect_create_data_ref_ptr (stmt_info, vectype,
+ ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
loop_for_initial_load, NULL_TREE,
- &init_addr, NULL, &inc, true, &inv_p);
+ &init_addr, NULL, &inc, true);
if (TREE_CODE (ptr) == SSA_NAME)
new_temp = copy_ssa_name (ptr);
else
new_temp = make_ssa_name (TREE_TYPE (ptr));
- unsigned int align = DR_TARGET_ALIGNMENT (dr);
+ poly_uint64 align = DR_TARGET_ALIGNMENT (dr_info);
+ tree type = TREE_TYPE (ptr);
new_stmt = gimple_build_assign
(new_temp, BIT_AND_EXPR, ptr,
- build_int_cst (TREE_TYPE (ptr), -(HOST_WIDE_INT) align));
+ fold_build2 (MINUS_EXPR, type,
+ build_int_cst (type, 0),
+ build_int_cst (type, align)));
new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
data_ref
if (!init_addr)
{
/* Generate the INIT_ADDR computation outside LOOP. */
- init_addr = vect_create_addr_base_for_vector_ref (stmt_info, &stmts,
+ init_addr = vect_create_addr_base_for_vector_ref (vinfo,
+ stmt_info, &stmts,
NULL_TREE);
if (loop)
{
4th vec (E4): 3 7 11 15 19 23 27 31. */
static void
-vect_permute_load_chain (vec<tree> dr_chain,
+vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_low);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
/* Create interleaving stmt (high part of):
high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
second_vect, perm3_mask_high);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[k] = data_ref;
}
}
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_even);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
/* data_ref = permute_odd (first_data_ref, second_data_ref); */
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, second_vect,
perm_mask_odd);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2+length/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
*/
static bool
-vect_shift_permute_load_chain (vec<tree> dr_chain,
+vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
unsigned int length,
stmt_vec_info stmt_info,
gimple_stmt_iterator *gsi,
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
unsigned int i;
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
unsigned HOST_WIDE_INT nelt, vf;
if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
first_vect, first_vect,
perm2_mask1);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[0] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
second_vect, second_vect,
perm2_mask2);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[1] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], shift1_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2 + length/2] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[0], vect[1], select_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[j/2] = data_ref;
}
memcpy (dr_chain.address (), result_chain->address (),
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
dr_chain[k], dr_chain[k],
perm3_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
vect[k % 3], vect[(k + 1) % 3],
shift1_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect_shift[k] = data_ref;
}
vect_shift[(4 - k) % 3],
vect_shift[(3 - k) % 3],
shift2_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
vect[k] = data_ref;
}
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
vect[0], shift3_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[nelt % 3] = data_ref;
data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
vect[1], shift4_mask);
- vect_finish_stmt_generation (stmt_info, perm_stmt, gsi);
+ vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
(*result_chain)[0] = data_ref;
return true;
}
*/
void
-vect_transform_grouped_load (stmt_vec_info stmt_info, vec<tree> dr_chain,
+vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
+ vec<tree> dr_chain,
int size, gimple_stmt_iterator *gsi)
{
machine_mode mode;
mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
|| pow2p_hwi (size)
- || !vect_shift_permute_load_chain (dr_chain, size, stmt_info,
+ || !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
gsi, &result_chain))
- vect_permute_load_chain (dr_chain, size, stmt_info, gsi, &result_chain);
- vect_record_grouped_load_vectors (stmt_info, result_chain);
+ vect_permute_load_chain (vinfo, dr_chain,
+ size, stmt_info, gsi, &result_chain);
+ vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
result_chain.release ();
}
for each vector to the associated scalar statement. */
void
-vect_record_grouped_load_vectors (stmt_vec_info stmt_info,
+vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info,
vec<tree> result_chain)
{
- vec_info *vinfo = stmt_info->vinfo;
stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
unsigned int i, gap_count;
tree tmp_data_ref;
correspond to the gaps. */
if (next_stmt_info != first_stmt_info
&& gap_count < DR_GROUP_GAP (next_stmt_info))
- {
- gap_count++;
- continue;
- }
+ {
+ gap_count++;
+ continue;
+ }
- while (next_stmt_info)
+ /* ??? The following needs cleanup after the removal of
+ DR_GROUP_SAME_DR_STMT. */
+ if (next_stmt_info)
{
stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref);
/* We assume that if VEC_STMT is not NULL, this is a case of multiple
STMT_VINFO_VEC_STMT (next_stmt_info) = new_stmt_info;
else
{
- if (!DR_GROUP_SAME_DR_STMT (next_stmt_info))
- {
- stmt_vec_info prev_stmt_info
- = STMT_VINFO_VEC_STMT (next_stmt_info);
- stmt_vec_info rel_stmt_info
- = STMT_VINFO_RELATED_STMT (prev_stmt_info);
- while (rel_stmt_info)
- {
- prev_stmt_info = rel_stmt_info;
- rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info);
- }
+ stmt_vec_info prev_stmt_info
+ = STMT_VINFO_VEC_STMT (next_stmt_info);
+ stmt_vec_info rel_stmt_info
+ = STMT_VINFO_RELATED_STMT (prev_stmt_info);
+ while (rel_stmt_info)
+ {
+ prev_stmt_info = rel_stmt_info;
+ rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info);
+ }
- STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
- }
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt_info;
}
next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
gap_count = 1;
- /* If NEXT_STMT_INFO accesses the same DR as the previous statement,
- put the same TMP_DATA_REF as its vectorized statement; otherwise
- get the next data-ref from RESULT_CHAIN. */
- if (!next_stmt_info || !DR_GROUP_SAME_DR_STMT (next_stmt_info))
- break;
}
}
}
on ALIGNMENT bit boundary. */
bool
-vect_can_force_dr_alignment_p (const_tree decl, unsigned int alignment)
+vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
{
if (!VAR_P (decl))
return false;
return false;
if (TREE_STATIC (decl))
- return (alignment <= MAX_OFILE_ALIGNMENT);
+ return (known_le (alignment,
+ (unsigned HOST_WIDE_INT) MAX_OFILE_ALIGNMENT));
else
- return (alignment <= MAX_STACK_ALIGNMENT);
+ return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
}
-/* Return whether the data reference DR is supported with respect to its
+/* Return whether the data reference DR_INFO is supported with respect to its
alignment.
If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
it is aligned, i.e., check if it is possible to vectorize it with different
alignment. */
enum dr_alignment_support
-vect_supportable_dr_alignment (struct data_reference *dr,
+vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
bool check_aligned_accesses)
{
- stmt_vec_info stmt_info = vect_dr_stmt (dr);
+ data_reference *dr = dr_info->dr;
+ stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
machine_mode mode = TYPE_MODE (vectype);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *vect_loop = NULL;
+ loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
+ class loop *vect_loop = NULL;
bool nested_in_vect_loop = false;
- if (aligned_access_p (dr) && !check_aligned_accesses)
+ if (aligned_access_p (dr_info) && !check_aligned_accesses)
return dr_aligned;
/* For now assume all conditional loads/stores support unaligned
else
return dr_explicit_realign_optimized;
}
- if (!known_alignment_for_access_p (dr))
+ if (!known_alignment_for_access_p (dr_info))
is_packed = not_size_aligned (DR_REF (dr));
if (targetm.vectorize.support_vector_misalignment
- (mode, type, DR_MISALIGNMENT (dr), is_packed))
+ (mode, type, DR_MISALIGNMENT (dr_info), is_packed))
/* Can't software pipeline the loads, but can at least do them. */
return dr_unaligned_supported;
}
bool is_packed = false;
tree type = (TREE_TYPE (DR_REF (dr)));
- if (!known_alignment_for_access_p (dr))
+ if (!known_alignment_for_access_p (dr_info))
is_packed = not_size_aligned (DR_REF (dr));
if (targetm.vectorize.support_vector_misalignment
- (mode, type, DR_MISALIGNMENT (dr), is_packed))
+ (mode, type, DR_MISALIGNMENT (dr_info), is_packed))
return dr_unaligned_supported;
}
projects / thirdparty / gcc.git / blobdiff