+2013-11-21 Richard Biener <rguenther@suse.de>
+
+ * tree-vect-loop-manip.c (vect_build_loop_niters,
+ vect_generate_tmps_on_preheader): Move ...
+ * tree-vect-loop.c (vect_build_loop_niters,
+ vect_generate_tmps_on_preheader): ... here and simplify.
+ (vect_transform_loop): Call them here and pass down results
+ to consumers.
+ * tree-vect-loop-manip.c (vect_do_peeling_for_loop_bound):
+ Get niter variables from caller.
+ (vect_do_peeling_for_alignment): Likewise.
+ * tree-vectorizer.h (vect_generate_tmps_on_preheader): Remove.
+ (vect_do_peeling_for_loop_bound, vect_do_peeling_for_alignment):
+ Adjust prototypes.
+
2013-11-21 Richard Biener <rguenther@suse.de>
* tree-ssa-loop-ch.c (copy_loop_headers): Decrement
}
-/* This function builds ni_name = number of iterations loop executes
- on the loop preheader. If SEQ is given the stmt is instead emitted
- there. */
-
-static tree
-vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq seq)
-{
- tree ni_name, var;
- gimple_seq stmts = NULL;
- edge pe;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
-
- var = create_tmp_var (TREE_TYPE (ni), "niters");
- ni_name = force_gimple_operand (ni, &stmts, false, var);
-
- pe = loop_preheader_edge (loop);
- if (stmts)
- {
- if (seq)
- gimple_seq_add_seq (&seq, stmts);
- else
- {
- basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
- gcc_assert (!new_bb);
- }
- }
-
- return ni_name;
-}
-
-
-/* This function generates the following statements:
-
- ni_name = number of iterations loop executes
- ratio = ni_name / vf
- ratio_mult_vf_name = ratio * vf
-
- and places them at the loop preheader edge or in COND_EXPR_STMT_LIST
- if that is non-NULL. */
-
-void
-vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
- tree *ni_name_ptr,
- tree *ratio_mult_vf_name_ptr,
- tree *ratio_name_ptr,
- gimple_seq cond_expr_stmt_list)
-{
-
- edge pe;
- basic_block new_bb;
- gimple_seq stmts;
- tree ni_name, ni_minus_gap_name;
- tree var;
- tree ratio_name;
- tree ratio_mult_vf_name;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- tree ni = LOOP_VINFO_NITERS (loop_vinfo);
- int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- tree log_vf;
-
- pe = loop_preheader_edge (loop);
-
- /* Generate temporary variable that contains
- number of iterations loop executes. */
-
- ni_name = vect_build_loop_niters (loop_vinfo, cond_expr_stmt_list);
- log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
-
- /* If epilogue loop is required because of data accesses with gaps, we
- subtract one iteration from the total number of iterations here for
- correct calculation of RATIO. */
- if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
- {
- ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name),
- ni_name,
- build_one_cst (TREE_TYPE (ni_name)));
- if (!is_gimple_val (ni_minus_gap_name))
- {
- var = create_tmp_var (TREE_TYPE (ni), "ni_gap");
-
- stmts = NULL;
- ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts,
- true, var);
- if (cond_expr_stmt_list)
- gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
- else
- {
- pe = loop_preheader_edge (loop);
- new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
- gcc_assert (!new_bb);
- }
- }
- }
- else
- ni_minus_gap_name = ni_name;
-
- /* Create: ratio = ni >> log2(vf) */
-
- ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name),
- ni_minus_gap_name, log_vf);
- if (!is_gimple_val (ratio_name))
- {
- var = create_tmp_var (TREE_TYPE (ni), "bnd");
-
- stmts = NULL;
- ratio_name = force_gimple_operand (ratio_name, &stmts, true, var);
- if (cond_expr_stmt_list)
- gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
- else
- {
- pe = loop_preheader_edge (loop);
- new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
- gcc_assert (!new_bb);
- }
- }
-
- /* Create: ratio_mult_vf = ratio << log2 (vf). */
-
- ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name),
- ratio_name, log_vf);
- if (!is_gimple_val (ratio_mult_vf_name))
- {
- var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
-
- stmts = NULL;
- ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts,
- true, var);
- if (cond_expr_stmt_list)
- gimple_seq_add_seq (&cond_expr_stmt_list, stmts);
- else
- {
- pe = loop_preheader_edge (loop);
- new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
- gcc_assert (!new_bb);
- }
- }
-
- *ni_name_ptr = ni_name;
- *ratio_mult_vf_name_ptr = ratio_mult_vf_name;
- *ratio_name_ptr = ratio_name;
-
- return;
-}
-
/* Function vect_can_advance_ivs_p
In case the number of iterations that LOOP iterates is unknown at compile
test. */
void
-vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
+vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo,
+ tree ni_name, tree ratio_mult_vf_name,
unsigned int th, bool check_profitability)
{
- tree ni_name, ratio_mult_vf_name;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
struct loop *new_loop;
edge update_e;
initialize_original_copy_tables ();
- /* Generate the following variables on the preheader of original loop:
-
- ni_name = number of iteration the original loop executes
- ratio = ni_name / vf
- ratio_mult_vf_name = ratio * vf */
- vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
- &ratio_mult_vf_name, ratio,
- cond_expr_stmt_list);
-
loop_num = loop->num;
new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
void
-vect_do_peeling_for_alignment (loop_vec_info loop_vinfo,
+vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, tree ni_name,
unsigned int th, bool check_profitability)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- tree niters_of_prolog_loop, ni_name;
+ tree niters_of_prolog_loop;
tree n_iters;
tree wide_prolog_niters;
struct loop *new_loop;
initialize_original_copy_tables ();
- ni_name = vect_build_loop_niters (loop_vinfo, NULL);
+ gimple_seq stmts = NULL;
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo,
ni_name,
&bound);
}
}
+
+/* This function builds ni_name = number of iterations. Statements
+ are queued onto SEQ. */
+
+static tree
+vect_build_loop_niters (loop_vec_info loop_vinfo, gimple_seq *seq)
+{
+ tree ni_name, var;
+ gimple_seq stmts = NULL;
+ tree ni = unshare_expr (LOOP_VINFO_NITERS (loop_vinfo));
+
+ var = create_tmp_var (TREE_TYPE (ni), "niters");
+ ni_name = force_gimple_operand (ni, &stmts, false, var);
+
+ if (stmts)
+ gimple_seq_add_seq (seq, stmts);
+
+ return ni_name;
+}
+
+
+/* This function generates the following statements:
+
+ ni_name = number of iterations loop executes
+ ratio = ni_name / vf
+ ratio_mult_vf_name = ratio * vf
+
+ and places them in COND_EXPR_STMT_LIST. */
+
+static void
+vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo,
+ tree ni_name,
+ tree *ratio_mult_vf_name_ptr,
+ tree *ratio_name_ptr,
+ gimple_seq *cond_expr_stmt_list)
+{
+ gimple_seq stmts;
+ tree ni_minus_gap_name;
+ tree var;
+ tree ratio_name;
+ tree ratio_mult_vf_name;
+ tree ni = LOOP_VINFO_NITERS (loop_vinfo);
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ tree log_vf;
+
+ log_vf = build_int_cst (TREE_TYPE (ni), exact_log2 (vf));
+
+ /* If epilogue loop is required because of data accesses with gaps, we
+ subtract one iteration from the total number of iterations here for
+ correct calculation of RATIO. */
+ if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
+ {
+ ni_minus_gap_name = fold_build2 (MINUS_EXPR, TREE_TYPE (ni_name),
+ ni_name,
+ build_one_cst (TREE_TYPE (ni_name)));
+ if (!is_gimple_val (ni_minus_gap_name))
+ {
+ var = create_tmp_var (TREE_TYPE (ni), "ni_gap");
+
+ stmts = NULL;
+ ni_minus_gap_name = force_gimple_operand (ni_minus_gap_name, &stmts,
+ true, var);
+ gimple_seq_add_seq (cond_expr_stmt_list, stmts);
+ }
+ }
+ else
+ ni_minus_gap_name = ni_name;
+
+ /* Create: ratio = ni >> log2(vf) */
+
+ ratio_name = fold_build2 (RSHIFT_EXPR, TREE_TYPE (ni_minus_gap_name),
+ ni_minus_gap_name, log_vf);
+ if (!is_gimple_val (ratio_name))
+ {
+ var = create_tmp_var (TREE_TYPE (ni), "bnd");
+
+ stmts = NULL;
+ ratio_name = force_gimple_operand (ratio_name, &stmts, true, var);
+ gimple_seq_add_seq (cond_expr_stmt_list, stmts);
+ }
+ *ratio_name_ptr = ratio_name;
+
+ /* Create: ratio_mult_vf = ratio << log2 (vf). */
+
+ if (ratio_mult_vf_name_ptr)
+ {
+ ratio_mult_vf_name = fold_build2 (LSHIFT_EXPR, TREE_TYPE (ratio_name),
+ ratio_name, log_vf);
+ if (!is_gimple_val (ratio_mult_vf_name))
+ {
+ var = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
+
+ stmts = NULL;
+ ratio_mult_vf_name = force_gimple_operand (ratio_mult_vf_name, &stmts,
+ true, var);
+ gimple_seq_add_seq (cond_expr_stmt_list, stmts);
+ }
+ *ratio_mult_vf_name_ptr = ratio_mult_vf_name;
+ }
+
+ return;
+}
+
+
/* Function vect_transform_loop.
The analysis phase has determined that the loop is vectorizable.
}
/* Peel the loop if there are data refs with unknown alignment.
- Only one data ref with unknown store is allowed. */
+ Only one data ref with unknown store is allowed.
+ This clobbers LOOP_VINFO_NITERS but retains the original
+ in LOOP_VINFO_NITERS_UNCHANGED. So we cannot avoid re-computing
+ niters. */
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
{
- vect_do_peeling_for_alignment (loop_vinfo, th, check_profitability);
+ gimple_seq stmts = NULL;
+ tree ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ vect_do_peeling_for_alignment (loop_vinfo, ni_name,
+ th, check_profitability);
check_profitability = false;
}
if ((int) tree_ctz (LOOP_VINFO_NITERS (loop_vinfo))
< exact_log2 (vectorization_factor)
|| LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
- vect_do_peeling_for_loop_bound (loop_vinfo, &ratio,
- th, check_profitability);
+ {
+ tree ni_name, ratio_mult_vf;
+ gimple_seq stmts = NULL;
+ ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
+ vect_generate_tmps_on_preheader (loop_vinfo, ni_name, &ratio_mult_vf,
+ &ratio, &stmts);
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
+ vect_do_peeling_for_loop_bound (loop_vinfo, ni_name, ratio_mult_vf,
+ th, check_profitability);
+ }
else if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
else
{
- tree ni_name, ratio_mult_vf;
- vect_generate_tmps_on_preheader (loop_vinfo, &ni_name, &ratio_mult_vf,
- &ratio, NULL);
+ tree ni_name;
+ gimple_seq stmts = NULL;
+ ni_name = vect_build_loop_niters (loop_vinfo, &stmts);
+ vect_generate_tmps_on_preheader (loop_vinfo, ni_name, NULL,
+ &ratio, &stmts);
+ gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop), stmts);
}
/* 1) Make sure the loop header has exactly two entries
extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, edge);
extern void vect_loop_versioning (loop_vec_info, unsigned int, bool);
-extern void vect_generate_tmps_on_preheader (loop_vec_info, tree *, tree *,
- tree *, gimple_seq);
-extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree *,
+extern void vect_do_peeling_for_loop_bound (loop_vec_info, tree, tree,
unsigned int, bool);
-extern void vect_do_peeling_for_alignment (loop_vec_info, unsigned int, bool);
+extern void vect_do_peeling_for_alignment (loop_vec_info, tree,
+ unsigned int, bool);
extern LOC find_loop_location (struct loop *);
extern bool vect_can_advance_ivs_p (loop_vec_info);
projects / thirdparty / gcc.git / commitdiff
