#include "cfglayout.h"
#include "expr.h"
#include "optabs.h"
+#include "toplev.h"
#include "tree-chrec.h"
#include "tree-data-ref.h"
#include "tree-scalar-evolution.h"
/* Main code transformation functions. */
static void vect_transform_loop (loop_vec_info, struct loops *);
-static void vect_transform_loop_bound (loop_vec_info);
+static void vect_transform_loop_bound (loop_vec_info, tree niters);
static bool vect_transform_stmt (tree, block_stmt_iterator *);
static bool vectorizable_load (tree, block_stmt_iterator *, tree *);
static bool vectorizable_store (tree, block_stmt_iterator *, tree *);
static bool vect_is_simple_iv_evolution (unsigned, tree, tree *, tree *, bool);
static void vect_mark_relevant (varray_type, tree);
static bool vect_stmt_relevant_p (tree, loop_vec_info);
-static tree vect_get_loop_niters (struct loop *, HOST_WIDE_INT *);
+static tree vect_get_loop_niters (struct loop *, tree *);
static bool vect_compute_data_ref_alignment
(struct data_reference *, loop_vec_info);
static bool vect_analyze_data_ref_access (struct data_reference *);
static bool vect_can_force_dr_alignment_p (tree, unsigned int);
static struct data_reference * vect_analyze_pointer_ref_access
(tree, tree, bool);
+static bool vect_analyze_loop_with_symbolic_num_of_iters (tree niters,
+ struct loop *loop);
static tree vect_get_base_and_bit_offset
(struct data_reference *, tree, tree, loop_vec_info, tree *, bool*);
static struct data_reference * vect_analyze_pointer_ref_access
static tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
static tree vect_get_vec_def_for_operand (tree, tree);
static tree vect_init_vector (tree, tree);
+static tree vect_build_symbol_bound (tree, int, struct loop *);
static void vect_finish_stmt_generation
(tree stmt, tree vec_stmt, block_stmt_iterator *bsi);
+static void vect_generate_tmps_on_preheader (loop_vec_info,
+ tree *, tree *,
+ tree *);
+static tree vect_build_loop_niters (loop_vec_info);
+static void vect_update_ivs_after_vectorizer (struct loop *, tree);
+
+/* Loop transformations prior to vectorizeration. */
+
+/* Loop transformations entry point function.
+ It can be used outside of the vectorizer
+ in case the loop to be manipulated answers conditions specified
+ in function documentation. */
+struct loop *tree_duplicate_loop_to_edge (struct loop *,
+ struct loops *, edge,
+ tree, tree, bool);
+
+static void allocate_new_names (bitmap);
+static void rename_use_op (use_operand_p);
+static void rename_def_op (def_operand_p, tree);
+static void rename_variables_in_bb (basic_block);
+static void free_new_names (bitmap);
+static void rename_variables_in_loop (struct loop *);
+static void copy_phi_nodes (struct loop *, struct loop *, bool);
+static void update_phis_for_duplicate_loop (struct loop *,
+ struct loop *,
+ bool after);
+static void update_phi_nodes_for_guard (edge, struct loop *);
+static void make_loop_iterate_ntimes (struct loop *, tree, tree, tree);
+static struct loop *tree_duplicate_loop_to_edge_cfg (struct loop *,
+ struct loops *,
+ edge);
+static edge add_loop_guard (basic_block, tree, basic_block);
+static bool verify_loop_for_duplication (struct loop *, bool, edge);
+
+/* Utilities dealing with loop peeling (not peeling itself). */
+static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);
+static void vect_update_niters_after_peeling (loop_vec_info, tree);
+static void vect_update_inits_of_dr (struct data_reference *, struct loop *,
+ tree niters);
+static void vect_update_inits_of_drs (loop_vec_info, tree);
+static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);
+
/* Utilities for creation and deletion of vec_info structs. */
loop_vec_info new_loop_vec_info (struct loop *loop);
void destroy_loop_vec_info (loop_vec_info);
stmt_vec_info new_stmt_vec_info (tree stmt, struct loop *loop);
-static bool vect_debug_stats (struct loop *loop);
-static bool vect_debug_details (struct loop *loop);
+static bool vect_debug_stats (struct loop *loop);
+static bool vect_debug_details (struct loop *loop);
+
+\f
+/* Utilities to support loop peeling for vectorization purposes. */
+
+
+/* For each definition in DEFINITIONS this function allocates
+ new ssa name. */
+
+static void
+allocate_new_names (bitmap definitions)
+{
+ unsigned ver;
+ bitmap_iterator bi;
+
+ EXECUTE_IF_SET_IN_BITMAP (definitions, 0, ver, bi)
+ {
+ tree def = ssa_name (ver);
+ tree *new_name_ptr = xmalloc (sizeof (tree));
+
+ bool abnormal = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def);
+
+ *new_name_ptr = duplicate_ssa_name (def, SSA_NAME_DEF_STMT (def));
+ SSA_NAME_OCCURS_IN_ABNORMAL_PHI (*new_name_ptr) = abnormal;
+
+ SSA_NAME_AUX (def) = new_name_ptr;
+ }
+}
+
+
+/* Renames the use *OP_P. */
+
+static void
+rename_use_op (use_operand_p op_p)
+{
+ tree *new_name_ptr;
+
+ if (TREE_CODE (USE_FROM_PTR (op_p)) != SSA_NAME)
+ return;
+
+ new_name_ptr = SSA_NAME_AUX (USE_FROM_PTR (op_p));
+
+ /* Something defined outside of the loop. */
+ if (!new_name_ptr)
+ return;
+
+ /* An ordinary ssa name defined in the loop. */
+
+ SET_USE (op_p, *new_name_ptr);
+}
+
+
+/* Renames the def *OP_P in statement STMT. */
+
+static void
+rename_def_op (def_operand_p op_p, tree stmt)
+{
+ tree *new_name_ptr;
+
+ if (TREE_CODE (DEF_FROM_PTR (op_p)) != SSA_NAME)
+ return;
+
+ new_name_ptr = SSA_NAME_AUX (DEF_FROM_PTR (op_p));
+
+ /* Something defined outside of the loop. */
+ if (!new_name_ptr)
+ return;
+
+ /* An ordinary ssa name defined in the loop. */
+
+ SET_DEF (op_p, *new_name_ptr);
+ SSA_NAME_DEF_STMT (DEF_FROM_PTR (op_p)) = stmt;
+}
+
+
+/* Renames the variables in basic block BB. */
+
+static void
+rename_variables_in_bb (basic_block bb)
+{
+ tree phi;
+ block_stmt_iterator bsi;
+ tree stmt;
+ stmt_ann_t ann;
+ use_optype uses;
+ vuse_optype vuses;
+ def_optype defs;
+ v_may_def_optype v_may_defs;
+ v_must_def_optype v_must_defs;
+ unsigned i;
+ edge e;
+ edge_iterator ei;
+
+ for (phi = phi_nodes (bb); phi; phi = TREE_CHAIN (phi))
+ rename_def_op (PHI_RESULT_PTR (phi), phi);
+
+ for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))
+ {
+ stmt = bsi_stmt (bsi);
+ get_stmt_operands (stmt);
+ ann = stmt_ann (stmt);
+
+ uses = USE_OPS (ann);
+ for (i = 0; i < NUM_USES (uses); i++)
+ rename_use_op (USE_OP_PTR (uses, i));
+
+ defs = DEF_OPS (ann);
+ for (i = 0; i < NUM_DEFS (defs); i++)
+ rename_def_op (DEF_OP_PTR (defs, i), stmt);
+
+ vuses = VUSE_OPS (ann);
+ for (i = 0; i < NUM_VUSES (vuses); i++)
+ rename_use_op (VUSE_OP_PTR (vuses, i));
+
+ v_may_defs = V_MAY_DEF_OPS (ann);
+ for (i = 0; i < NUM_V_MAY_DEFS (v_may_defs); i++)
+ {
+ rename_use_op (V_MAY_DEF_OP_PTR (v_may_defs, i));
+ rename_def_op (V_MAY_DEF_RESULT_PTR (v_may_defs, i), stmt);
+ }
+
+ v_must_defs = V_MUST_DEF_OPS (ann);
+ for (i = 0; i < NUM_V_MUST_DEFS (v_must_defs); i++)
+ rename_def_op (V_MUST_DEF_OP_PTR (v_must_defs, i), stmt);
+ }
+
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ for (phi = phi_nodes (e->dest); phi; phi = TREE_CHAIN (phi))
+ rename_use_op (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e));
+}
+
+
+/* Releases the structures holding the new ssa names. */
+
+static void
+free_new_names (bitmap definitions)
+{
+ unsigned ver;
+ bitmap_iterator bi;
+
+ EXECUTE_IF_SET_IN_BITMAP (definitions, 0, ver, bi)
+ {
+ tree def = ssa_name (ver);
+
+ if (SSA_NAME_AUX (def))
+ {
+ free (SSA_NAME_AUX (def));
+ SSA_NAME_AUX (def) = NULL;
+ }
+ }
+}
+
+
+/* Renames variables in new generated LOOP. */
+
+static void
+rename_variables_in_loop (struct loop *loop)
+{
+ unsigned i;
+ basic_block *bbs;
+
+ bbs = get_loop_body (loop);
+
+ for (i = 0; i < loop->num_nodes; i++)
+ rename_variables_in_bb (bbs[i]);
+
+ free (bbs);
+}
+
+
+/* This function copies phis from LOOP header to
+ NEW_LOOP header. AFTER is as
+ in update_phis_for_duplicate_loop function. */
+
+static void
+copy_phi_nodes (struct loop *loop, struct loop *new_loop,
+ bool after)
+{
+ tree phi, new_phi, def;
+ edge new_e;
+ edge e = (after ? loop_latch_edge (loop) : loop_preheader_edge (loop));
+
+ /* Second add arguments to newly created phi nodes. */
+ for (phi = phi_nodes (loop->header),
+ new_phi = phi_nodes (new_loop->header);
+ phi;
+ phi = TREE_CHAIN (phi),
+ new_phi = TREE_CHAIN (new_phi))
+ {
+ new_e = loop_preheader_edge (new_loop);
+ def = PHI_ARG_DEF_FROM_EDGE (phi, e);
+ add_phi_arg (&new_phi, def, new_e);
+ }
+}
+
+
+/* Update the PHI nodes of the NEW_LOOP. AFTER is true if the NEW_LOOP
+ executes after LOOP, and false if it executes before it. */
+
+static void
+update_phis_for_duplicate_loop (struct loop *loop,
+ struct loop *new_loop, bool after)
+{
+ edge old_latch;
+ tree *new_name_ptr, new_ssa_name;
+ tree phi_new, phi_old, def;
+ edge orig_entry_e = loop_preheader_edge (loop);
+
+ /* Copy phis from loop->header to new_loop->header. */
+ copy_phi_nodes (loop, new_loop, after);
+
+ old_latch = loop_latch_edge (loop);
+
+ /* Update PHI args for the new loop latch edge, and
+ the old loop preheader edge, we know that the PHI nodes
+ are ordered appropriately in copy_phi_nodes. */
+ for (phi_new = phi_nodes (new_loop->header),
+ phi_old = phi_nodes (loop->header);
+ phi_new && phi_old;
+ phi_new = TREE_CHAIN (phi_new), phi_old = TREE_CHAIN (phi_old))
+ {
+ def = PHI_ARG_DEF_FROM_EDGE (phi_old, old_latch);
+
+ if (TREE_CODE (def) != SSA_NAME)
+ continue;
+
+ new_name_ptr = SSA_NAME_AUX (def);
+
+ /* Something defined outside of the loop. */
+ if (!new_name_ptr)
+ continue;
+
+ /* An ordinary ssa name defined in the loop. */
+ new_ssa_name = *new_name_ptr;
+
+ add_phi_arg (&phi_new, new_ssa_name, loop_latch_edge(new_loop));
+
+ /* Update PHI args for the original loop pre-header edge. */
+ if (! after)
+ SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi_old, orig_entry_e),
+ new_ssa_name);
+ }
+}
+
+
+/* Update PHI nodes for a guard of the LOOP.
+
+ LOOP is supposed to have a preheader bb at which a guard condition is
+ located. The true edge of this condition skips the LOOP and ends
+ at the destination of the (unique) LOOP exit. The loop exit bb is supposed
+ to be an empty bb (created by this transformation) with one successor.
+
+ This function creates phi nodes at the LOOP exit bb. These phis need to be
+ created as a result of adding true edge coming from guard.
+
+ FORNOW: Only phis which have corresponding phi nodes at the header of the
+ LOOP are created. Here we use the assumption that after the LOOP there
+ are no uses of defs generated in LOOP.
+
+ After the phis creation, the function updates the values of phi nodes at
+ the LOOP exit successor bb:
+
+ Original loop:
+
+ bb0: loop preheader
+ goto bb1
+ bb1: loop header
+ if (exit_cond) goto bb3 else goto bb2
+ bb2: loop latch
+ goto bb1
+ bb3:
+
+
+ After guard creation (the loop before this function):
+
+ bb0: loop preheader
+ if (guard_condition) goto bb4 else goto bb1
+ bb1: loop header
+ if (exit_cond) goto bb4 else goto bb2
+ bb2: loop latch
+ goto bb1
+ bb4: loop exit
+ (new empty bb)
+ goto bb3
+ bb3:
+
+ This function updates the phi nodes in bb4 and in bb3, to account for the
+ new edge from bb0 to bb4. */
+
+static void
+update_phi_nodes_for_guard (edge guard_true_edge, struct loop * loop)
+{
+ tree phi, phi1;
+
+ for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ {
+ tree new_phi;
+ tree phi_arg;
+
+ /* Generate new phi node. */
+ new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (phi)),
+ loop->exit_edges[0]->dest);
+
+ /* Add argument coming from guard true edge. */
+ phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, loop->entry_edges[0]);
+ add_phi_arg (&new_phi, phi_arg, guard_true_edge);
+
+ /* Add argument coming from loop exit edge. */
+ phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, EDGE_SUCC (loop->latch, 0));
+ add_phi_arg (&new_phi, phi_arg, loop->exit_edges[0]);
+
+ /* Update all phi nodes at the loop exit successor. */
+ for (phi1 = phi_nodes (EDGE_SUCC (loop->exit_edges[0]->dest, 0)->dest);
+ phi1;
+ phi1 = TREE_CHAIN (phi1))
+ {
+ tree old_arg = PHI_ARG_DEF_FROM_EDGE (phi1,
+ EDGE_SUCC (loop->exit_edges[0]->dest, 0));
+ if (old_arg == phi_arg)
+ {
+ edge e = EDGE_SUCC (loop->exit_edges[0]->dest, 0);
+
+ SET_PHI_ARG_DEF (phi1,
+ phi_arg_from_edge (phi1, e),
+ PHI_RESULT (new_phi));
+ }
+ }
+ }
+}
+
+
+/* Make the LOOP iterate NITERS times. This is done by adding a new IV
+ that starts at zero, increases by one and its limit is NITERS. */
+
+static void
+make_loop_iterate_ntimes (struct loop *loop, tree niters,
+ tree begin_label, tree exit_label)
+{
+ tree indx_before_incr, indx_after_incr, cond_stmt, cond;
+ tree orig_cond;
+ edge exit_edge = loop->exit_edges[0];
+ block_stmt_iterator loop_exit_bsi = bsi_last (exit_edge->src);
+
+ /* Flow loop scan does not update loop->single_exit field. */
+ loop->single_exit = loop->exit_edges[0];
+ orig_cond = get_loop_exit_condition (loop);
+ gcc_assert (orig_cond);
+ create_iv (integer_zero_node, integer_one_node, NULL_TREE, loop,
+ &loop_exit_bsi, false, &indx_before_incr, &indx_after_incr);
+
+ /* CREATE_IV uses BSI_INSERT with TSI_NEW_STMT, so we want to get
+ back to the exit condition statement. */
+ bsi_next (&loop_exit_bsi);
+ gcc_assert (bsi_stmt (loop_exit_bsi) == orig_cond);
+
+
+ if (exit_edge->flags & EDGE_TRUE_VALUE) /* 'then' edge exits the loop. */
+ cond = build2 (GE_EXPR, boolean_type_node, indx_after_incr, niters);
+ else /* 'then' edge loops back. */
+ cond = build2 (LT_EXPR, boolean_type_node, indx_after_incr, niters);
+
+ begin_label = build1 (GOTO_EXPR, void_type_node, begin_label);
+ exit_label = build1 (GOTO_EXPR, void_type_node, exit_label);
+ cond_stmt = build (COND_EXPR, TREE_TYPE (orig_cond), cond,
+ begin_label, exit_label);
+ bsi_insert_before (&loop_exit_bsi, cond_stmt, BSI_SAME_STMT);
+
+ /* Remove old loop exit test: */
+ bsi_remove (&loop_exit_bsi);
+
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ print_generic_expr (dump_file, cond_stmt, TDF_SLIM);
+}
+
+
+/* Given LOOP this function generates a new copy of it and puts it
+ on E which is either the entry or exit of LOOP. */
+
+static struct loop *
+tree_duplicate_loop_to_edge_cfg (struct loop *loop, struct loops *loops,
+ edge e)
+{
+ struct loop *new_loop;
+ basic_block *new_bbs, *bbs;
+ bool at_exit;
+ bool was_imm_dom;
+ basic_block exit_dest;
+ tree phi, phi_arg;
+
+ at_exit = (e == loop->exit_edges[0]);
+ if (!at_exit && e != loop_preheader_edge (loop))
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ "Edge is not an entry nor an exit edge.\n");
+ return NULL;
+ }
+
+ bbs = get_loop_body (loop);
+
+ /* Check whether duplication is possible. */
+ if (!can_copy_bbs_p (bbs, loop->num_nodes))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "Cannot copy basic blocks.\n");
+ free (bbs);
+ return NULL;
+ }
+
+ /* Generate new loop structure. */
+ new_loop = duplicate_loop (loops, loop, loop->outer);
+ if (!new_loop)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "The duplicate_loop returns NULL.\n");
+ free (bbs);
+ return NULL;
+ }
+
+ exit_dest = loop->exit_edges[0]->dest;
+ was_imm_dom = (get_immediate_dominator (CDI_DOMINATORS,
+ exit_dest) == loop->header ?
+ true : false);
+
+ new_bbs = xmalloc (sizeof (basic_block) * loop->num_nodes);
+
+ copy_bbs (bbs, loop->num_nodes, new_bbs, NULL, 0, NULL, NULL);
+
+ /* Duplicating phi args at exit bbs as coming
+ also from exit of duplicated loop. */
+ for (phi = phi_nodes (exit_dest); phi; phi = TREE_CHAIN (phi))
+ {
+ phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, loop->exit_edges[0]);
+ if (phi_arg)
+ {
+ edge new_loop_exit_edge;
+
+ if (EDGE_SUCC (new_loop->header, 0)->dest == new_loop->latch)
+ new_loop_exit_edge = EDGE_SUCC (new_loop->header, 1);
+ else
+ new_loop_exit_edge = EDGE_SUCC (new_loop->header, 0);
+
+ add_phi_arg (&phi, phi_arg, new_loop_exit_edge);
+ }
+ }
+
+ if (at_exit) /* Add the loop copy at exit. */
+ {
+ redirect_edge_and_branch_force (e, new_loop->header);
+ set_immediate_dominator (CDI_DOMINATORS, new_loop->header, e->src);
+ if (was_imm_dom)
+ set_immediate_dominator (CDI_DOMINATORS, exit_dest, new_loop->header);
+ }
+ else /* Add the copy at entry. */
+ {
+ edge new_exit_e;
+ edge entry_e = loop_preheader_edge (loop);
+ basic_block preheader = entry_e->src;
+
+ if (!flow_bb_inside_loop_p (new_loop,
+ EDGE_SUCC (new_loop->header, 0)->dest))
+ new_exit_e = EDGE_SUCC (new_loop->header, 0);
+ else
+ new_exit_e = EDGE_SUCC (new_loop->header, 1);
+
+ redirect_edge_and_branch_force (new_exit_e, loop->header);
+ set_immediate_dominator (CDI_DOMINATORS, loop->header,
+ new_exit_e->src);
+
+ /* We have to add phi args to the loop->header here as coming
+ from new_exit_e edge. */
+ for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ {
+ phi_arg = PHI_ARG_DEF_FROM_EDGE (phi, entry_e);
+ if (phi_arg)
+ add_phi_arg (&phi, phi_arg, new_exit_e);
+ }
+
+ redirect_edge_and_branch_force (entry_e, new_loop->header);
+ set_immediate_dominator (CDI_DOMINATORS, new_loop->header, preheader);
+ }
+
+ flow_loop_scan (new_loop, LOOP_ALL);
+ flow_loop_scan (loop, LOOP_ALL);
+ free (new_bbs);
+ free (bbs);
+
+ return new_loop;
+}
+
+
+/* Given the condition statement COND, put it as the last statement
+ of GUARD_BB; EXIT_BB is the basic block to skip the loop;
+ Assumes that this is the single exit of the guarded loop.
+ Returns the skip edge. */
+
+static edge
+add_loop_guard (basic_block guard_bb, tree cond, basic_block exit_bb)
+{
+ block_stmt_iterator bsi;
+ edge new_e, enter_e;
+ tree cond_stmt, then_label, else_label;
+
+ enter_e = EDGE_SUCC (guard_bb, 0);
+ enter_e->flags &= ~EDGE_FALLTHRU;
+ enter_e->flags |= EDGE_FALSE_VALUE;
+ bsi = bsi_last (guard_bb);
+
+ then_label = build1 (GOTO_EXPR, void_type_node,
+ tree_block_label (exit_bb));
+ else_label = build1 (GOTO_EXPR, void_type_node,
+ tree_block_label (enter_e->dest));
+ cond_stmt = build (COND_EXPR, void_type_node, cond,
+ then_label, else_label);
+ bsi_insert_after (&bsi, cond_stmt, BSI_NEW_STMT);
+ /* Add new edge to connect entry block to the second loop. */
+ new_e = make_edge (guard_bb, exit_bb, EDGE_TRUE_VALUE);
+ set_immediate_dominator (CDI_DOMINATORS, exit_bb, guard_bb);
+ return new_e;
+}
+
+
+/* This function verifies that certain restrictions apply to LOOP. */
+
+static bool
+verify_loop_for_duplication (struct loop *loop,
+ bool update_first_loop_count, edge e)
+{
+ edge exit_e = loop->exit_edges [0];
+ edge entry_e = loop_preheader_edge (loop);
+
+ /* We duplicate only innermost loops. */
+ if (loop->inner)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "Loop duplication failed. Loop is not innermost.\n");
+ return false;
+ }
+
+ /* Only loops with 1 exit. */
+ if (loop->num_exits != 1)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "More than one exit from loop.\n");
+ return false;
+ }
+
+ /* Only loops with 1 entry. */
+ if (loop->num_entries != 1)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "More than one exit from loop.\n");
+ return false;
+ }
+
+ /* All loops has outers, the only case loop->outer is NULL is for
+ the function itself. */
+ if (!loop->outer)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "Loop is outer-most loop.\n");
+ return false;
+ }
+
+ /* Verify that new IV can be created and loop condition
+ can be changed to make first loop iterate first_niters times. */
+ if (!update_first_loop_count)
+ {
+ tree orig_cond = get_loop_exit_condition (loop);
+ block_stmt_iterator loop_exit_bsi = bsi_last (exit_e->src);
+
+ if (!orig_cond)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "Loop has no exit condition.\n");
+ return false;
+ }
+ if (orig_cond != bsi_stmt (loop_exit_bsi))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "Loop exit condition is not loop header last stmt.\n");
+ return false;
+ }
+ }
+
+ /* Make sure E is either an entry or an exit edge. */
+ if (e != exit_e && e != entry_e)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "E is not loop entry or exit edge.\n");
+ return false;
+ }
+
+ return true;
+}
+
+
+/* Given LOOP this function duplicates it to the edge E.
+
+ This transformation takes place before the loop is vectorized.
+ For now, there are two main cases when it's used
+ by the vectorizer: to support loops with unknown loop bounds
+ (or loop bounds indivisible by vectorization factor) and to force the
+ alignment of data references in the loop. In the first case, LOOP is
+ duplicated to the exit edge, producing epilog loop. In the second case, LOOP
+ is duplicated to the preheader edge thus generating prolog loop. In both
+ cases, the original loop will be vectorized after the transformation.
+
+ The edge E is supposed to be either preheader edge of the LOOP or
+ its exit edge. If preheader edge is specified, the LOOP copy
+ will precede the original one. Otherwise the copy will be located
+ at the exit of the LOOP.
+
+ FIRST_NITERS (SSA_NAME) parameter specifies how many times to iterate
+ the first loop. If UPDATE_FIRST_LOOP_COUNT parameter is false, the first
+ loop will be iterated FIRST_NITERS times by introducing additional
+ induction variable and replacing loop exit condition. If
+ UPDATE_FIRST_LOOP_COUNT is true no change to the first loop is made and
+ the caller to tree_duplicate_loop_to_edge is responsible for updating
+ the first loop count.
+
+ NITERS (also SSA_NAME) parameter defines the number of iteration the
+ original loop iterated. The function generates two if-then guards:
+ one prior to the first loop and the other prior to the second loop.
+ The first guard will be:
+
+ if (FIRST_NITERS == 0) then skip the first loop
+
+ The second guard will be:
+
+ if (FIRST_NITERS == NITERS) then skip the second loop
+
+ Thus the equivalence to the original code is guaranteed by correct values
+ of NITERS and FIRST_NITERS and generation of if-then loop guards.
+
+ For now this function supports only loop forms that are candidate for
+ vectorization. Such types are the following:
+
+ (1) only innermost loops
+ (2) loops built from 2 basic blocks
+ (3) loops with one entry and one exit
+ (4) loops without function calls
+ (5) loops without defs that are used after the loop
+
+ (1), (3) are checked in this function; (2) - in function
+ vect_analyze_loop_form; (4) - in function vect_analyze_data_refs;
+ (5) is checked as part of the function vect_mark_stmts_to_be_vectorized,
+ when excluding induction/reduction support.
+
+ The function returns NULL in case one of these checks or
+ transformations failed. */
+
+struct loop*
+tree_duplicate_loop_to_edge (struct loop *loop, struct loops *loops,
+ edge e, tree first_niters,
+ tree niters, bool update_first_loop_count)
+{
+ struct loop *new_loop = NULL, *first_loop, *second_loop;
+ edge skip_e;
+ tree pre_condition;
+ bitmap definitions;
+ basic_block first_exit_bb, second_exit_bb;
+ basic_block pre_header_bb;
+ edge exit_e = loop->exit_edges [0];
+
+ gcc_assert (!any_marked_for_rewrite_p ());
+
+ if (!verify_loop_for_duplication (loop, update_first_loop_count, e))
+ return NULL;
+
+ /* We have to initialize cfg_hooks. Then, when calling
+ cfg_hooks->split_edge, the function tree_split_edge
+ is actually called and, when calling cfg_hooks->duplicate_block,
+ the function tree_duplicate_bb is called. */
+ tree_register_cfg_hooks ();
+
+ /* 1. Generate a copy of LOOP and put it on E (entry or exit). */
+ if (!(new_loop = tree_duplicate_loop_to_edge_cfg (loop, loops, e)))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "The tree_duplicate_loop_to_edge_cfg failed.\n");
+ return NULL;
+ }
+
+ definitions = marked_ssa_names ();
+ allocate_new_names (definitions);
+ update_phis_for_duplicate_loop (loop, new_loop, e == exit_e);
+ /* Here, using assumption (5), we do not propagate new names futher
+ than on phis of the exit from the second loop. */
+ rename_variables_in_loop (new_loop);
+ free_new_names (definitions);
+
+ if (e == exit_e)
+ {
+ first_loop = loop;
+ second_loop = new_loop;
+ }
+ else
+ {
+ first_loop = new_loop;
+ second_loop = loop;
+ }
+
+ /* 2. Generate bb between the loops. */
+ first_exit_bb = split_edge (first_loop->exit_edges[0]);
+ add_bb_to_loop (first_exit_bb, first_loop->outer);
+
+ /* We need to update here first loop exit edge
+ and second loop preheader edge. */
+ flow_loop_scan (first_loop, LOOP_ALL);
+ flow_loop_scan (second_loop, LOOP_ALL);
+
+ /* 3. Make first loop iterate FIRST_NITERS times, if needed. */
+ if (!update_first_loop_count)
+ {
+ tree first_loop_latch_lbl = tree_block_label (first_loop->latch);
+ tree first_loop_exit_lbl = tree_block_label (first_exit_bb);
+
+ make_loop_iterate_ntimes (first_loop, first_niters,
+ first_loop_latch_lbl,
+ first_loop_exit_lbl);
+ }
+
+ /* 4. Add the guard before first loop:
+
+ if FIRST_NITERS == 0
+ skip first loop
+ else
+ enter first loop */
+
+ /* 4a. Generate bb before first loop. */
+ pre_header_bb = split_edge (loop_preheader_edge (first_loop));
+ add_bb_to_loop (pre_header_bb, first_loop->outer);
+
+ /* First loop preheader edge is changed. */
+ flow_loop_scan (first_loop, LOOP_ALL);
+ /* 4b. Generate guard condition. */
+ pre_condition = build (LE_EXPR, boolean_type_node,
+ first_niters, integer_zero_node);
+
+ /* 4c. Add condition at the end of preheader bb. */
+ skip_e = add_loop_guard (pre_header_bb, pre_condition, first_exit_bb);
+
+ /* 4d. Updtae phis at first loop exit and propagate changes
+ to the phis of second loop. */
+ update_phi_nodes_for_guard (skip_e, first_loop);
+
+ /* 5. Add the guard before second loop:
+
+ if FIRST_NITERS == NITERS SKIP
+ skip second loop
+ else
+ enter second loop */
+
+ /* 5a. Generate empty bb at the exit from the second loop. */
+ second_exit_bb = split_edge (second_loop->exit_edges[0]);
+ add_bb_to_loop (second_exit_bb, second_loop->outer);
+
+ /* Second loop preheader edge is changed. */
+ flow_loop_scan (second_loop, LOOP_ALL);
+
+ /* 5b. Generate guard condition. */
+ pre_condition = build (EQ_EXPR, boolean_type_node,
+ first_niters, niters);
+
+ /* 5c. Add condition at the end of preheader bb. */
+ skip_e = add_loop_guard (first_exit_bb, pre_condition, second_exit_bb);
+ update_phi_nodes_for_guard (skip_e, second_loop);
+
+ BITMAP_XFREE (definitions);
+ unmark_all_for_rewrite ();
+
+ return new_loop;
+}
+
+
+\f
+/* Here the proper Vectorizer starts. */
/* Function new_stmt_vec_info.
LOOP_VINFO_LOOP (res) = loop;
LOOP_VINFO_BBS (res) = bbs;
LOOP_VINFO_EXIT_COND (res) = NULL;
- LOOP_VINFO_NITERS (res) = -1;
+ LOOP_VINFO_NITERS (res) = NULL;
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
+ LOOP_DO_PEELING_FOR_ALIGNMENT (res) = false;
LOOP_VINFO_VECT_FACTOR (res) = 0;
VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DATAREF_WRITES (res), 20,
"loop_write_datarefs");
VARRAY_GENERIC_PTR_INIT (LOOP_VINFO_DATAREF_READS (res), 20,
"loop_read_datarefs");
+
+ for (i=0; i<MAX_NUMBER_OF_UNALIGNED_DATA_REFS; i++)
+ LOOP_UNALIGNED_DR (res, i) = NULL;
return res;
}
}
-
/* Function vect_force_dr_alignment_p.
Returns whether the alignment of a DECL can be forced to be aligned
OFFSET: Optional. If supplied, it is be added to the initial address.
Output:
- 1. Return an SSA_NAME whose value is the address of the memory location of the
- first vector of the data reference.
+ 1. Return an SSA_NAME whose value is the address of the memory location of
+ the first vector of the data reference.
2. If new_stmt_list is not NULL_TREE after return then the caller must insert
these statement(s) which define the returned SSA_NAME.
/* Only the access function of the last index is relevant (i_n in
a[i_1][i_2]...[i_n]), the others correspond to loop invariants. */
access_fn = DR_ACCESS_FN (dr, 0);
- ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_oval, &step, true);
+ ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_oval, &step,
+ true);
if (!ok)
init_oval = integer_zero_node;
if (!STMT_VINFO_DATA_REF (stmt_info))
return false;
- if (!aligned_access_p (STMT_VINFO_DATA_REF (stmt_info)))
- return false;
if (!vec_stmt) /* transformation not required. */
{
software_pipeline_loads_p = true;
else if (!targetm.vectorize.misaligned_mem_ok (mode))
{
- /* Possibly unaligned access, and can't software pipeline the loads */
+ /* Possibly unaligned access, and can't software pipeline the loads.
+ */
if (vect_debug_details (loop))
fprintf (dump_file, "Arbitrary load not supported.");
return false;
}
else
{
- /* Use current address instead of init_addr for reduced reg pressure. */
+ /* Use current address instead of init_addr for reduced reg pressure.
+ */
magic = dataref_ptr;
}
add_phi_arg (&phi_stmt, lsq, loop_latch_edge (loop));
- /* <5> Create <vec_dest = realign_load (msq, lsq, magic)> in loop */
- vec_dest = vect_create_destination_var (scalar_dest, vectype);
- new_stmt = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, magic);
- new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
- new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
- vect_finish_stmt_generation (stmt, new_stmt, bsi);
- }
+ /* <5> Create <vec_dest = realign_load (msq, lsq, magic)> in loop */
+ vec_dest = vect_create_destination_var (scalar_dest, vectype);
+ new_stmt = build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, magic);
+ new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ TREE_OPERAND (new_stmt, 0) = new_temp;
+ vect_finish_stmt_generation (stmt, new_stmt, bsi);
+ }
+
+ *vec_stmt = new_stmt;
+ return true;
+}
+
+
+/* Function vect_transform_stmt.
+
+ Create a vectorized stmt to replace STMT, and insert it at BSI. */
+
+static bool
+vect_transform_stmt (tree stmt, block_stmt_iterator *bsi)
+{
+ bool is_store = false;
+ tree vec_stmt = NULL_TREE;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ bool done;
+
+ switch (STMT_VINFO_TYPE (stmt_info))
+ {
+ case op_vec_info_type:
+ done = vectorizable_operation (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case assignment_vec_info_type:
+ done = vectorizable_assignment (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case load_vec_info_type:
+ done = vectorizable_load (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case store_vec_info_type:
+ done = vectorizable_store (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ is_store = true;
+ break;
+ default:
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "stmt not supported.");
+ gcc_unreachable ();
+ }
+
+ STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
+
+ return is_store;
+}
+
+
+/* This function builds ni_name = number of iterations loop executes
+ on the loop preheader. */
+
+static tree
+vect_build_loop_niters (loop_vec_info loop_vinfo)
+{
+ tree ni_name, stmt, var;
+ edge pe;
+ basic_block new_bb;
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree ni = unshare_expr (LOOP_VINFO_NITERS(loop_vinfo));
+
+ var = create_tmp_var (TREE_TYPE (ni), "niters");
+ add_referenced_tmp_var (var);
+ if (TREE_CODE (ni) == INTEGER_CST)
+ {
+ /* This case is generated when treating a known loop bound
+ indivisible by VF. Here we cannot use force_gimple_operand. */
+ stmt = build (MODIFY_EXPR, void_type_node, var, ni);
+ ni_name = make_ssa_name (var, stmt);
+ TREE_OPERAND (stmt, 0) = ni_name;
+ }
+ else
+ ni_name = force_gimple_operand (ni, &stmt, false, var);
+
+ pe = loop_preheader_edge (loop);
+ new_bb = bsi_insert_on_edge_immediate (pe, stmt);
+ if (new_bb)
+ add_bb_to_loop (new_bb, EDGE_PRED (new_bb, 0)->src->loop_father);
+
+ return ni_name;
+}
+
+
+/* This function generates the following statements:
+
+ ni_name = number of iterations loop executes
+ ratio = ni_name / vf
+ ratio_mult_vf_name = ratio * vf
+
+ and places them at the loop preheader edge. */
+
+static void
+vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo, tree *ni_name_p,
+ tree *ratio_mult_vf_name_p, tree *ratio_p)
+{
+
+ edge pe;
+ basic_block new_bb;
+ tree stmt, ni_name;
+ tree ratio;
+ tree ratio_mult_vf_name, ratio_mult_vf;
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree ni = LOOP_VINFO_NITERS(loop_vinfo);
+
+ int vf, i;
+
+ /* Generate temporary variable that contains
+ number of iterations loop executes. */
+
+ ni_name = vect_build_loop_niters (loop_vinfo);
+
+ /* ratio = ni / vf.
+ vf is power of 2; then if ratio = = n >> log2 (vf). */
+ vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ ratio = vect_build_symbol_bound (ni_name, vf, loop);
+
+ /* Update initial conditions of loop copy. */
+
+ /* ratio_mult_vf = ratio * vf;
+ then if ratio_mult_vf = ratio << log2 (vf). */
+
+ i = exact_log2 (vf);
+ ratio_mult_vf = create_tmp_var (TREE_TYPE (ni), "ratio_mult_vf");
+ add_referenced_tmp_var (ratio_mult_vf);
+
+ ratio_mult_vf_name = make_ssa_name (ratio_mult_vf, NULL_TREE);
+
+ stmt = build2 (MODIFY_EXPR, void_type_node, ratio_mult_vf_name,
+ build2 (LSHIFT_EXPR, TREE_TYPE (ratio),
+ ratio, build_int_cst (unsigned_type_node,
+ i)));
+
+ SSA_NAME_DEF_STMT (ratio_mult_vf_name) = stmt;
+
+ pe = loop_preheader_edge (loop);
+ new_bb = bsi_insert_on_edge_immediate (pe, stmt);
+ if (new_bb)
+ add_bb_to_loop (new_bb, EDGE_PRED (new_bb, 0)->src->loop_father);
+
+ *ni_name_p = ni_name;
+ *ratio_mult_vf_name_p = ratio_mult_vf_name;
+ *ratio_p = ratio;
+
+ return;
+}
+
+
+/* This function generates stmt
+
+ tmp = n / vf;
+
+ and attaches it to preheader of LOOP. */
+
+static tree
+vect_build_symbol_bound (tree n, int vf, struct loop * loop)
+{
+ tree var, stmt, var_name;
+ edge pe;
+ basic_block new_bb;
+ int i;
+
+ /* create temporary variable */
+ var = create_tmp_var (TREE_TYPE (n), "bnd");
+ add_referenced_tmp_var (var);
+
+ var_name = make_ssa_name (var, NULL_TREE);
+
+ /* vf is power of 2; then n/vf = n >> log2 (vf). */
+
+ i = exact_log2 (vf);
+ stmt = build2 (MODIFY_EXPR, void_type_node, var_name,
+ build2 (RSHIFT_EXPR, TREE_TYPE (n),
+ n, build_int_cst (unsigned_type_node,i)));
+
+ SSA_NAME_DEF_STMT (var_name) = stmt;
+
+ pe = loop_preheader_edge (loop);
+ new_bb = bsi_insert_on_edge_immediate (pe, stmt);
+ if (new_bb)
+ add_bb_to_loop (new_bb, EDGE_PRED (new_bb, 0)->src->loop_father);
+ else
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "New bb on preheader edge was not generated.");
+
+ return var_name;
+}
+
+
+/* Function vect_transform_loop_bound.
+
+ Create a new exit condition for the loop. */
+
+static void
+vect_transform_loop_bound (loop_vec_info loop_vinfo, tree niters)
+{
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ edge exit_edge = loop->single_exit;
+ block_stmt_iterator loop_exit_bsi = bsi_last (exit_edge->src);
+ tree indx_before_incr, indx_after_incr;
+ tree orig_cond_expr;
+ HOST_WIDE_INT old_N = 0;
+ int vf;
+ tree cond_stmt;
+ tree new_loop_bound;
+ bool symbol_niters;
+ tree cond;
+ tree lb_type;
+
+ symbol_niters = !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo);
+
+ if (!symbol_niters)
+ old_N = LOOP_VINFO_INT_NITERS (loop_vinfo);
+
+ vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+
+ orig_cond_expr = LOOP_VINFO_EXIT_COND (loop_vinfo);
+#ifdef ENABLE_CHECKING
+ gcc_assert (orig_cond_expr);
+#endif
+ gcc_assert (orig_cond_expr == bsi_stmt (loop_exit_bsi));
+
+ create_iv (integer_zero_node, integer_one_node, NULL_TREE, loop,
+ &loop_exit_bsi, false, &indx_before_incr, &indx_after_incr);
+
+ /* bsi_insert is using BSI_NEW_STMT. We need to bump it back
+ to point to the exit condition. */
+ bsi_next (&loop_exit_bsi);
+ gcc_assert (bsi_stmt (loop_exit_bsi) == orig_cond_expr);
+
+ /* new loop exit test: */
+ lb_type = TREE_TYPE (TREE_OPERAND (TREE_OPERAND (orig_cond_expr, 0), 1));
+ if (!symbol_niters)
+ new_loop_bound = fold_convert (lb_type,
+ build_int_cst (unsigned_type_node,
+ old_N/vf));
+ else
+ new_loop_bound = niters;
+
+ if (exit_edge->flags & EDGE_TRUE_VALUE) /* 'then' edge exits the loop. */
+ cond = build2 (GE_EXPR, boolean_type_node,
+ indx_after_incr, new_loop_bound);
+ else /* 'then' edge loops back. */
+ cond = build2 (LT_EXPR, boolean_type_node,
+ indx_after_incr, new_loop_bound);
+
+ cond_stmt = build3 (COND_EXPR, TREE_TYPE (orig_cond_expr), cond,
+ TREE_OPERAND (orig_cond_expr, 1), TREE_OPERAND (orig_cond_expr, 2));
+
+ bsi_insert_before (&loop_exit_bsi, cond_stmt, BSI_SAME_STMT);
+
+ /* remove old loop exit test: */
+ bsi_remove (&loop_exit_bsi);
+
+ if (vect_debug_details (NULL))
+ print_generic_expr (dump_file, cond_stmt, TDF_SLIM);
+}
+
+
+/* Advance IVs of the loop (to be vectorized later) to correct position.
+
+ When loop is vectorized, its IVs are not always advanced
+ correctly since vectorization changes the loop count. It's ok
+ in case epilog loop was not produced after original one before
+ vectorization process (the vectorizer checks that there is no uses
+ of IVs after the loop). However, in case the epilog loop was peeled,
+ IVs from original loop are used in epilog loop and should be
+ advanced correctly.
+
+ Here we use access functions of IVs and number of
+ iteration loop executes in order to bring IVs to correct position.
+
+ Function also update phis of basic block at the exit
+ from the loop. */
+
+static void
+vect_update_ivs_after_vectorizer (struct loop *loop, tree niters)
+{
+ edge exit = loop->exit_edges[0];
+ tree phi;
+ edge latch = loop_latch_edge (loop);
+
+ /* Generate basic block at the exit from the loop. */
+ basic_block new_bb = split_edge (exit);
+ add_bb_to_loop (new_bb, EDGE_SUCC (new_bb, 0)->dest->loop_father);
+
+ loop->exit_edges[0] = EDGE_PRED (new_bb, 0);
+
+ for (phi = phi_nodes (loop->header); phi; phi = TREE_CHAIN (phi))
+ {
+ tree access_fn = NULL;
+ tree evolution_part;
+ tree init_expr;
+ tree step_expr;
+ tree var, stmt, ni, ni_name;
+ int i, j, num_elem1, num_elem2;
+ tree phi1;
+ block_stmt_iterator last_bsi;
+
+ /* Skip virtual phi's. The data dependences that are associated with
+ virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
+
+ if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi))))
+ {
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "virtual phi. skip.");
+ continue;
+ }
+
+ access_fn = analyze_scalar_evolution (loop, PHI_RESULT (phi));
+
+ evolution_part = evolution_part_in_loop_num (access_fn, loop->num);
+
+ /* FORNOW: We do not transform initial conditions of IVs
+ which evolution functions are a polynomial of degree >= 2 or
+ exponential. */
+
+ step_expr = evolution_part;
+ init_expr = initial_condition (access_fn);
+
+ ni = build2 (PLUS_EXPR, TREE_TYPE (init_expr),
+ build2 (MULT_EXPR, TREE_TYPE (niters),
+ niters, step_expr), init_expr);
+
+ var = create_tmp_var (TREE_TYPE (init_expr), "tmp");
+ add_referenced_tmp_var (var);
+
+ ni_name = force_gimple_operand (ni, &stmt, false, var);
+
+ /* Insert stmt into new_bb. */
+ last_bsi = bsi_last (new_bb);
+ bsi_insert_after (&last_bsi, stmt, BSI_NEW_STMT);
+
+ /* Fix phi expressions in duplicated loop. */
+ num_elem1 = PHI_NUM_ARGS (phi);
+ for (i = 0; i < num_elem1; i++)
+ if (PHI_ARG_EDGE (phi, i) == latch)
+ {
+ tree def = PHI_ARG_DEF (phi, i);
+
+ for (phi1 = phi_nodes (EDGE_SUCC (new_bb, 0)->dest); phi1;
+ phi1 = TREE_CHAIN (phi1))
+ {
+ num_elem2 = PHI_NUM_ARGS (phi1);
+ for (j = 0; j < num_elem2; j++)
+ if (PHI_ARG_DEF (phi1, j) == def)
+ {
+ SET_PHI_ARG_DEF (phi1, j, ni_name);
+ PHI_ARG_EDGE (phi1, j) = EDGE_SUCC (new_bb, 0);
+ break;
+ }
+ }
+ break;
+ }
+ }
+
+}
+
+
+/* This function is the main driver of tranformation
+ to be done for loop before vectorizing it in case of
+ unknown loop bound. */
+
+static void
+vect_transform_for_unknown_loop_bound (loop_vec_info loop_vinfo, tree * ratio,
+ struct loops *loops)
+{
+
+ tree ni_name, ratio_mult_vf_name;
+#ifdef ENABLE_CHECKING
+ int loop_num;
+#endif
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *new_loop;
+
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "\n<<vect_transtorm_for_unknown_loop_bound>>\n");
+
+ /* Generate the following variables on the preheader of original loop:
+
+ ni_name = number of iteration the original loop executes
+ ratio = ni_name / vf
+ ratio_mult_vf_name = ratio * vf */
+ vect_generate_tmps_on_preheader (loop_vinfo, &ni_name,
+ &ratio_mult_vf_name, ratio);
+
+ /* Update loop info. */
+ loop->pre_header = loop_preheader_edge (loop)->src;
+ loop->pre_header_edges[0] = loop_preheader_edge (loop);
+
+#ifdef ENABLE_CHECKING
+ loop_num = loop->num;
+#endif
+ new_loop = tree_duplicate_loop_to_edge (loop, loops, loop->exit_edges[0],
+ ratio_mult_vf_name, ni_name, true);
+#ifdef ENABLE_CHECKING
+ gcc_assert (new_loop);
+ gcc_assert (loop_num == loop->num);
+#endif
+
+ /* Update IVs of original loop as if they were advanced
+ by ratio_mult_vf_name steps. */
+
+#ifdef ENABLE_CHECKING
+ /* Check existence of intermediate bb. */
+ gcc_assert (loop->exit_edges[0]->dest == new_loop->pre_header);
+#endif
+ vect_update_ivs_after_vectorizer (loop, ratio_mult_vf_name);
+
+ return;
- *vec_stmt = new_stmt;
- return true;
}
-/* Function vect_transform_stmt.
+/* Function vect_gen_niters_for_prolog_loop
- Create a vectorized stmt to replace STMT, and insert it at BSI. */
+ Set the number of iterations for the loop represented by LOOP_VINFO
+ to the minimum between NITERS (the original iteration count of the loop)
+ and the misalignment DR - the first data reference in the list
+ LOOP_UNALIGNED_DR (LOOP_VINFO). As a result, after the execution of this
+ loop, the data reference DR will refer to an aligned location. */
-static bool
-vect_transform_stmt (tree stmt, block_stmt_iterator *bsi)
+static tree
+vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo, tree niters)
{
- bool is_store = false;
- tree vec_stmt = NULL_TREE;
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- bool done;
+ struct data_reference *dr = LOOP_UNALIGNED_DR (loop_vinfo, 0);
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree var, stmt;
+ tree iters, iters_name;
+ edge pe;
+ basic_block new_bb;
+ tree dr_stmt = DR_STMT (dr);
+ stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
+ tree start_addr, byte_miss_align, elem_miss_align;
+ int vec_type_align =
+ GET_MODE_ALIGNMENT (TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info)))
+ / BITS_PER_UNIT;
+ tree tmp1, tmp2;
+ tree new_stmt_list = NULL_TREE;
- switch (STMT_VINFO_TYPE (stmt_info))
- {
- case op_vec_info_type:
- done = vectorizable_operation (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
+ start_addr = vect_create_addr_base_for_vector_ref (dr_stmt,
+ &new_stmt_list, NULL_TREE);
+
+ pe = loop_preheader_edge (loop);
+ new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);
+ if (new_bb)
+ add_bb_to_loop (new_bb, EDGE_PRED (new_bb, 0)->src->loop_father);
+
+ byte_miss_align =
+ build (BIT_AND_EXPR, integer_type_node, start_addr,
+ build (MINUS_EXPR, integer_type_node,
+ build_int_cst (unsigned_type_node,
+ vec_type_align), integer_one_node));
+ tmp1 = build_int_cst (unsigned_type_node, vec_type_align/vf);
+ elem_miss_align = build (FLOOR_DIV_EXPR, integer_type_node,
+ byte_miss_align, tmp1);
+
+ tmp2 =
+ build (BIT_AND_EXPR, integer_type_node,
+ build (MINUS_EXPR, integer_type_node,
+ build_int_cst (unsigned_type_node, vf), elem_miss_align),
+ build (MINUS_EXPR, integer_type_node,
+ build_int_cst (unsigned_type_node, vf), integer_one_node));
+
+ iters = build2 (MIN_EXPR, TREE_TYPE (tmp2), tmp2, niters);
+ var = create_tmp_var (TREE_TYPE (iters), "iters");
+ add_referenced_tmp_var (var);
+ iters_name = force_gimple_operand (iters, &stmt, false, var);
+
+ /* Insert stmt on loop preheader edge. */
+ pe = loop_preheader_edge (loop);
+ new_bb = bsi_insert_on_edge_immediate (pe, stmt);
+ if (new_bb)
+ add_bb_to_loop (new_bb, EDGE_PRED (new_bb, 0)->src->loop_father);
- case assignment_vec_info_type:
- done = vectorizable_assignment (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
+ return iters_name;
+}
- case load_vec_info_type:
- done = vectorizable_load (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
- case store_vec_info_type:
- done = vectorizable_store (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- is_store = true;
- break;
- default:
- if (vect_debug_details (NULL))
- fprintf (dump_file, "stmt not supported.");
- gcc_unreachable ();
- }
+/* Function vect_update_niters_after_peeling
- STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
+ NITERS iterations were peeled from the loop represented by LOOP_VINFO.
+ The new number of iterations is therefore original_niters - NITERS.
+ Record the new number of iterations in LOOP_VINFO. */
- return is_store;
+static void
+vect_update_niters_after_peeling (loop_vec_info loop_vinfo, tree niters)
+{
+ tree n_iters = LOOP_VINFO_NITERS (loop_vinfo);
+ LOOP_VINFO_NITERS (loop_vinfo) =
+ build (MINUS_EXPR, integer_type_node, n_iters, niters);
}
-/* Function vect_transform_loop_bound.
+/* Function vect_update_inits_of_dr
- Create a new exit condition for the loop. */
+ NITERS iterations were peeled from LOOP. DR represents a data reference
+ in LOOP. This function updates the information recorded in DR to
+ account for the fact that the first NITERS iterations had already been
+ executed. Specifically, it updates the initial_condition of the
+ access_function of DR. */
static void
-vect_transform_loop_bound (loop_vec_info loop_vinfo)
+vect_update_inits_of_dr (struct data_reference *dr, struct loop *loop,
+ tree niters)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- edge exit_edge = loop->single_exit;
- block_stmt_iterator loop_exit_bsi = bsi_last (exit_edge->src);
- tree indx_before_incr, indx_after_incr;
- tree orig_cond_expr;
- HOST_WIDE_INT old_N = 0;
- int vf;
- tree cond_stmt;
- tree new_loop_bound;
- tree cond;
- tree lb_type;
+ tree access_fn = DR_ACCESS_FN (dr, 0);
+ tree init, init_new, step;
+
+ step = evolution_part_in_loop_num (access_fn, loop->num);
+ init = initial_condition (access_fn);
+
+ init_new = build (PLUS_EXPR, TREE_TYPE (init),
+ build (MULT_EXPR, TREE_TYPE (niters),
+ niters, step), init);
+ DR_ACCESS_FN (dr, 0) = chrec_replace_initial_condition (access_fn, init_new);
+
+ return;
+}
- gcc_assert (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo));
- old_N = LOOP_VINFO_NITERS (loop_vinfo);
- vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- /* FORNOW:
- assuming number-of-iterations divides by the vectorization factor. */
- gcc_assert (!(old_N % vf));
+/* Function vect_update_inits_of_drs
- orig_cond_expr = LOOP_VINFO_EXIT_COND (loop_vinfo);
- gcc_assert (orig_cond_expr);
- gcc_assert (orig_cond_expr == bsi_stmt (loop_exit_bsi));
+ NITERS iterations were peeled from the loop represented by LOOP_VINFO.
+ This function updates the information recorded for the data references in
+ the loop to account for the fact that the first NITERS iterations had
+ already been executed. Specifically, it updates the initial_condition of the
+ access_function of all the data_references in the loop. */
- create_iv (integer_zero_node, integer_one_node, NULL_TREE, loop,
- &loop_exit_bsi, false, &indx_before_incr, &indx_after_incr);
+static void
+vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters)
+{
+ unsigned int i;
+ varray_type loop_write_datarefs = LOOP_VINFO_DATAREF_WRITES (loop_vinfo);
+ varray_type loop_read_datarefs = LOOP_VINFO_DATAREF_READS (loop_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- /* bsi_insert is using BSI_NEW_STMT. We need to bump it back
- to point to the exit condition. */
- bsi_next (&loop_exit_bsi);
- gcc_assert (bsi_stmt (loop_exit_bsi) == orig_cond_expr);
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, "\n<<vect_update_inits_of_dr>>\n");
- /* new loop exit test: */
- lb_type = TREE_TYPE (TREE_OPERAND (TREE_OPERAND (orig_cond_expr, 0), 1));
- new_loop_bound = build_int_cst (lb_type, old_N/vf);
+ for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_write_datarefs); i++)
+ {
+ struct data_reference *dr = VARRAY_GENERIC_PTR (loop_write_datarefs, i);
+ vect_update_inits_of_dr (dr, loop, niters);
+ }
- if (exit_edge->flags & EDGE_TRUE_VALUE) /* 'then' edge exits the loop. */
- cond = build2 (GE_EXPR, boolean_type_node, indx_after_incr, new_loop_bound);
- else /* 'then' edge loops back. */
- cond = build2 (LT_EXPR, boolean_type_node, indx_after_incr, new_loop_bound);
+ for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_read_datarefs); i++)
+ {
+ struct data_reference *dr = VARRAY_GENERIC_PTR (loop_read_datarefs, i);
+ vect_update_inits_of_dr (dr, loop, niters);
+ DR_MISALIGNMENT (dr) = -1;
+ }
+}
- cond_stmt = build3 (COND_EXPR, TREE_TYPE (orig_cond_expr), cond,
- TREE_OPERAND (orig_cond_expr, 1), TREE_OPERAND (orig_cond_expr, 2));
- bsi_insert_before (&loop_exit_bsi, cond_stmt, BSI_SAME_STMT);
+/* Function vect_do_peeling_for_alignment
- /* remove old loop exit test: */
- bsi_remove (&loop_exit_bsi);
+ Peel the first 'niters' iterations of the loop represented by LOOP_VINFO.
+ 'niters' is set to the misalignment of one of the data references in the
+ loop, thereby forcing it to refer to an aligned location at the beginning
+ of the execution of this loop. The data reference for which we are
+ peeling is chosen from LOOP_UNALIGNED_DR. */
+
+static void
+vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, struct loops *loops)
+{
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree niters_of_prolog_loop, ni_name;
+ struct data_reference *dr = LOOP_UNALIGNED_DR (loop_vinfo, 0);
if (vect_debug_details (NULL))
- print_generic_expr (dump_file, cond_stmt, TDF_SLIM);
+ fprintf (dump_file, "\n<<vect_do_peeling_for_alignment>>\n");
+
+ ni_name = vect_build_loop_niters (loop_vinfo);
+ niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
+
+
+ /* Peel the prolog loop and iterate it niters_of_prolog_loop. */
+ tree_duplicate_loop_to_edge (loop, loops, loop_preheader_edge(loop),
+ niters_of_prolog_loop, ni_name, false);
+
+
+ /* Update stmt info of dr according to which we peeled. */
+ DR_MISALIGNMENT (dr) = 0;
+
+ /* Update number of times loop executes. */
+ vect_update_niters_after_peeling (loop_vinfo, niters_of_prolog_loop);
+
+ /* Update all inits of access functions of all data refs. */
+ vect_update_inits_of_drs (loop_vinfo, niters_of_prolog_loop);
+
+ /* After peeling we have to reset scalar evolution analyzer. */
+ scev_reset ();
+
+ return;
}
int nbbs = loop->num_nodes;
block_stmt_iterator si;
int i;
+ tree ratio = NULL;
#ifdef ENABLE_CHECKING
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
#endif
if (vect_debug_details (NULL))
fprintf (dump_file, "\n<<vec_transform_loop>>\n");
+
+ /* Peel the loop if there are data refs with unknown alignment.
+ Only one data ref with unknown store is allowed. */
+
+
+ if (LOOP_DO_PEELING_FOR_ALIGNMENT (loop_vinfo))
+ vect_do_peeling_for_alignment (loop_vinfo, loops);
+
+ /* If the loop has a symbolic number of iterations 'n'
+ (i.e. it's not a compile time constant),
+ then an epilog loop needs to be created. We therefore duplicate
+ the initial loop. The original loop will be vectorized, and will compute
+ the first (n/VF) iterations. The second copy of the loop will remain
+ serial and will compute the remaining (n%VF) iterations.
+ (VF is the vectorization factor). */
+
+ if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
+ vect_transform_for_unknown_loop_bound (loop_vinfo, &ratio, loops);
+
+ /* FORNOW: we'll treat the case where niters is constant and
+
+ niters % vf != 0
+
+ in the way similar to one with symbolic niters.
+ For this we'll generate variable which value is equal to niters. */
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && (LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0))
+ vect_transform_for_unknown_loop_bound (loop_vinfo, &ratio, loops);
+
+
/* 1) Make sure the loop header has exactly two entries
2) Make sure we have a preheader basic block. */
} /* stmts in BB */
} /* BBs in loop */
- vect_transform_loop_bound (loop_vinfo);
+ vect_transform_loop_bound (loop_vinfo, ratio);
if (vect_debug_details (loop))
fprintf (dump_file,"Success! loop vectorized.");
}
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
- /* FORNOW: handle only cases where the loop bound divides by the
- vectorization factor. */
-
- if (vect_debug_details (NULL))
+
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && vect_debug_details (NULL))
fprintf (dump_file,
"vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC,
- vectorization_factor, LOOP_VINFO_NITERS (loop_vinfo));
-
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
- {
- if (vect_debug_stats (loop) || vect_debug_details (loop))
- fprintf (dump_file, "not vectorized: Unknown loop bound.");
- return false;
- }
+ vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
- if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && LOOP_VINFO_NITERS (loop_vinfo) % vectorization_factor != 0)
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ && LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0)
{
- if (vect_debug_stats (loop) || vect_debug_details (loop))
- fprintf (dump_file, "not vectorized: loop bound doesn't divided by %d.",
- vectorization_factor);
- return false;
+ /* In this case we have to generate epilog loop, that
+ can be done only for loops with one entry edge. */
+ if (LOOP_VINFO_LOOP (loop_vinfo)->num_entries != 1
+ || !(LOOP_VINFO_LOOP (loop_vinfo)->pre_header))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file, "not vectorized: more than one entry.");
+ return false;
+ }
}
-
+
return true;
}
return false;
step_expr = evolution_part;
- init_expr = initial_condition (access_fn);
+ init_expr = unshare_expr (initial_condition (access_fn));
if (vect_debug_details (NULL))
{
tree nbits;
if (TREE_CODE (TREE_TYPE (ref)) == ARRAY_TYPE)
- /* The reference is an array without its last index. */
- next_ref = vect_compute_array_base_alignment (ref, vectype, &dims, &misalign);
+ /* The reference is an array without its last index. */
+ next_ref = vect_compute_array_base_alignment (ref, vectype, &dims,
+ &misalign);
else
- next_ref =
- vect_compute_array_base_alignment (oprnd0, vectype, &dims, &misalign);
+ next_ref = vect_compute_array_base_alignment (oprnd0, vectype, &dims,
+ &misalign);
if (!vectype)
/* Alignment is not requested. Just return the base. */
return next_ref;
vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
{
varray_type loop_write_datarefs = LOOP_VINFO_DATAREF_WRITES (loop_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
/*varray_type loop_read_datarefs = LOOP_VINFO_DATAREF_READS (loop_vinfo);*/
unsigned int i;
+ unsigned int decide_peeling_count = 0;
if (vect_debug_details (NULL))
fprintf (dump_file, "\n<<vect_analyze_data_refs_alignment>>\n");
/* Finally, check that loop can be vectorized.
- FOR NOW: Until support for misaligned accesses is in place, only if all
- accesses are aligned can the loop be vectorized. This restriction will be
- relaxed. */
+ FOR NOW: Until support for misaligned stores is in place, only if all
+ stores are aligned can the loop be vectorized. This restriction will be
+ relaxed. In the meantime, we can force the alignment of on of the
+ data-references in the loop using peeling. We currently use a heuristic
+ that peels the first misaligned store, but we plan to develop a
+ better cost model to guide the decision on which data-access to peel for.
+ */
for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_write_datarefs); i++)
{
struct data_reference *dr = VARRAY_GENERIC_PTR (loop_write_datarefs, i);
if (!aligned_access_p (dr))
{
- if (vect_debug_stats (LOOP_VINFO_LOOP (loop_vinfo))
- || vect_debug_details (LOOP_VINFO_LOOP (loop_vinfo)))
- fprintf (dump_file, "not vectorized: unaligned store.");
- return false;
+ /* Decide here whether we need peeling for alignment. */
+ decide_peeling_count++;
+ if (decide_peeling_count > MAX_NUMBER_OF_UNALIGNED_DATA_REFS)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "not vectorized: multiple misaligned stores.");
+ return false;
+ }
+ else
+ {
+ LOOP_UNALIGNED_DR (loop_vinfo, decide_peeling_count - 1) = dr;
+ LOOP_DO_PEELING_FOR_ALIGNMENT (loop_vinfo) = true;
+ }
}
}
if (evolution_part_in_loop_num (access_fn,
loop_containing_stmt (DR_STMT (dr))->num))
{
- /* Evolution part is not NULL in this loop (it is neither constant nor
- invariant). */
+ /* Evolution part is not NULL in this loop (it is neither constant
+ nor invariant). */
if (vect_debug_details (NULL))
{
fprintf (dump_file,
- "not vectorized: complicated multidimensional array access.");
+ "not vectorized: complicated multidim. array access.");
print_generic_expr (dump_file, access_fn, TDF_SLIM);
}
return false;
{
if (vect_debug_details (NULL))
{
- fprintf (dump_file, "not vectorized: too complicated access function.");
+ fprintf (dump_file, "not vectorized: complicated access function.");
print_generic_expr (dump_file, access_fn, TDF_SLIM);
}
return false;
/* Analyze MEMREF. If it is of a supported form, build data_reference
struct for it (DR) and find the relevant symbol for aliasing
purposes. */
- symbl = vect_get_symbl_and_dr (memref, stmt, is_read, loop_vinfo, &dr);
+ symbl = vect_get_symbl_and_dr (memref, stmt, is_read, loop_vinfo,
+ &dr);
if (!symbl)
{
if (vect_debug_stats (loop) || vect_debug_details (loop))
switch (TREE_CODE (address_base))
{
case ARRAY_REF:
- dr = analyze_array (stmt, TREE_OPERAND (symbl, 0), DR_IS_READ(dr));
+ dr = analyze_array (stmt, TREE_OPERAND (symbl, 0),
+ DR_IS_READ(dr));
STMT_VINFO_MEMTAG (stmt_info) =
vect_get_base_and_bit_offset (dr, DR_BASE_NAME (dr), NULL_TREE,
loop_vinfo, &offset,
default:
if (vect_debug_stats (loop) || vect_debug_details (loop))
{
- fprintf (dump_file, "not vectorized: unhandled address expression: ");
+ fprintf (dump_file,
+ "not vectorized: unhandled address expr: ");
print_generic_expr (dump_file, stmt, TDF_SLIM);
}
return false;
}
+/* Function vect_analyze_loop_with_symbolic_num_of_iters.
+
+ In case the number of iterations that LOOP iterates in unknown at compile
+ time, an epilog loop will be generated, and the loop induction variables
+ (IVs) will be "advanced" to the value they are supposed to take just before
+ the epilog loop. Here we check that the access function of the loop IVs
+ and the expression that represents the loop bound are simple enough.
+ These restrictions will be relaxed in the future. */
+
+static bool
+vect_analyze_loop_with_symbolic_num_of_iters (tree niters,
+ struct loop *loop)
+{
+ basic_block bb = loop->header;
+ tree phi;
+
+ if (vect_debug_details (NULL))
+ fprintf (dump_file,
+ "\n<<vect_analyze_loop_with_symbolic_num_of_iters>>\n");
+
+ if (chrec_contains_undetermined (niters))
+ {
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "Infinite number of iterations.");
+ return false;
+ }
+
+ if (!niters)
+ {
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "niters is NULL pointer.");
+ return false;
+ }
+
+ if (vect_debug_details (NULL))
+ {
+ fprintf (dump_file, "Symbolic number of iterations is ");
+ print_generic_expr (dump_file, niters, TDF_DETAILS);
+ }
+
+ /* Analyze phi functions of the loop header. */
+
+ for (phi = phi_nodes (bb); phi; phi = TREE_CHAIN (phi))
+ {
+ tree access_fn = NULL;
+ tree evolution_part;
+
+ if (vect_debug_details (NULL))
+ {
+ fprintf (dump_file, "Analyze phi: ");
+ print_generic_expr (dump_file, phi, TDF_SLIM);
+ }
+
+ /* Skip virtual phi's. The data dependences that are associated with
+ virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
+
+ if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi))))
+ {
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "virtual phi. skip.");
+ continue;
+ }
+
+ /* Analyze the evolution function. */
+
+ access_fn = instantiate_parameters
+ (loop, analyze_scalar_evolution (loop, PHI_RESULT (phi)));
+
+ if (!access_fn)
+ {
+ if (vect_debug_details (NULL))
+ fprintf (dump_file, "No Access function.");
+ return false;
+ }
+
+ if (vect_debug_details (NULL))
+ {
+ fprintf (dump_file, "Access function of PHI: ");
+ print_generic_expr (dump_file, access_fn, TDF_SLIM);
+ }
+
+ evolution_part = evolution_part_in_loop_num (access_fn, loop->num);
+
+ if (evolution_part == NULL_TREE)
+ return false;
+
+ /* FORNOW: We do not transform initial conditions of IVs
+ which evolution functions are a polynomial of degree >= 2. */
+
+ if (tree_is_chrec (evolution_part))
+ return false;
+ }
+
+ return true;
+}
+
+
/* Function vect_get_loop_niters.
Determine how many iterations the loop is executed. */
static tree
-vect_get_loop_niters (struct loop *loop, HOST_WIDE_INT *number_of_iterations)
+vect_get_loop_niters (struct loop *loop, tree *number_of_iterations)
{
tree niters;
niters = number_of_iterations_in_loop (loop);
if (niters != NULL_TREE
- && niters != chrec_dont_know
- && host_integerp (niters,0))
+ && niters != chrec_dont_know)
{
- *number_of_iterations = TREE_INT_CST_LOW (niters);
+ *number_of_iterations = niters;
if (vect_debug_details (NULL))
- fprintf (dump_file, "==> get_loop_niters:" HOST_WIDE_INT_PRINT_DEC,
- *number_of_iterations);
+ {
+ fprintf (dump_file, "==> get_loop_niters:" );
+ print_generic_expr (dump_file, *number_of_iterations, TDF_SLIM);
+ }
}
return get_loop_exit_condition (loop);
{
loop_vec_info loop_vinfo;
tree loop_cond;
- HOST_WIDE_INT number_of_iterations = -1;
+ tree number_of_iterations = NULL;
if (vect_debug_details (loop))
fprintf (dump_file, "\n<<vect_analyze_loop_form>>\n");
fprintf (dump_file, "not vectorized: complicated exit condition.");
return NULL;
}
-
- if (number_of_iterations < 0)
+
+ if (!number_of_iterations)
{
if (vect_debug_stats (loop) || vect_debug_details (loop))
- fprintf (dump_file, "not vectorized: unknown loop bound.");
+ fprintf (dump_file,
+ "not vectorized: number of iterations cannot be computed.");
return NULL;
}
- if (number_of_iterations == 0) /* CHECKME: can this happen? */
+ loop_vinfo = new_loop_vec_info (loop);
+ LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
+ if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file, "loop bound unknown.");
+
+ /* Unknown loop bound. */
+ if (!vect_analyze_loop_with_symbolic_num_of_iters
+ (number_of_iterations, loop))
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "not vectorized: can't determine loop bound.");
+ return NULL;
+ }
+ else
+ {
+ /* We need only one loop entry for unknown loop bound support. */
+ if (loop->num_entries != 1 || !loop->pre_header)
+ {
+ if (vect_debug_stats (loop) || vect_debug_details (loop))
+ fprintf (dump_file,
+ "not vectorized: more than one loop entry.");
+ return NULL;
+ }
+ }
+ }
+ else
+ if (LOOP_VINFO_INT_NITERS (loop_vinfo) == 0)
{
if (vect_debug_stats (loop) || vect_debug_details (loop))
fprintf (dump_file, "not vectorized: number of iterations = 0.");
return NULL;
}
- loop_vinfo = new_loop_vec_info (loop);
LOOP_VINFO_EXIT_COND (loop_vinfo) = loop_cond;
- LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
return loop_vinfo;
}
Information in virtual phi nodes is sufficient for it. */
rewrite_into_loop_closed_ssa ();
}
+ rewrite_into_loop_closed_ssa ();
bitmap_clear (vars_to_rename);
}
projects / thirdparty / gcc.git / commitdiff
