vect: Enable prolog peeling for uncounted loops

The categorization of uncounted loops as
LOOP_VINFO_EARLY_BREAKS_VECT_PEELED disables prolog peeling by
default.  This is due to the assumption that you have early break
exits following the IV counting main exit.  For such loops, prolog
peeling is indeed problematic.

For enabling prolog peeling in uncounted loops it is sufficient, when
duplicating the loop for the prolog, to convert the prolog loop into a
counted loop, inserting a counting IV exit at the end, thus resulting
in the kind of early-break loop already supported by the compiler.

The pre-existing exits will continue to point to the exit node, while
the new exit will point to the vectorized loop, directing control flow
there once the number of iterations required for alignment are
completed.

In order to achieve this, we note that `vect_set_loop_condition'
replaces the condition in the main exit of a counted loop, all the
while inserting the prolog IV and its update statement.  The design
strategy is thus:

  - Have `slpeel_tree_duplicate_loop_to_edge_cfg' add a dummy main
  exit to the loop in the non-exiting branch of the original "main"
  exit in the loop, between the condition-containing BB and the latch
  BB.  For the original exit, if the exit condition is true, the
  edge->dest will remain unchanged.  The dummy exit will replicate
  this control-flow, with the exiting branch of the if statement
  initially leading to the same exit BB as the preceding exit.

  - As this new basic block will contain the IV-counting exit
  condition, its exit edge will be used for the control flow when
  alignment is achieved and thus we mark it as the new `new_exit'.
  This exit is then used in `redirect_edge_and_branch_force (new_exit,
  preheader)' and its basic block passed to `vect_set_loop_condition',
  wherein its condition will be replaced accordingly, correctly
  completing the setting up of our prolog loop.

  - In order to control this new functionality in
  slpeel_tree_duplicate_loop_to_edge_cfg we are, however, required to
  add a new parameter to the function. This is to be set to true when
  we have an uncounted loop AND we're generating its prolog.  This is
  done via the `bool duplicate_main_e' parameter, defaulting to false,
  allowing existing calls to the function to remain unchanged.

gcc/ChangeLog:

	* tree-vect-data-refs.cc (vect_enhance_data_refs_alignment):
	Enable peeling for uncounted loops.
	* tree-vect-loop-manip.cc
	(slpeel_tree_duplicate_loop_to_edge_cfg): Add exit condition
	duplication functionality.
	* tree-vectorizer.h (slpeel_tree_duplicate_loop_to_edge_cfg):
	Modify function signature.
	(vect_do_peeling): Enable uncounted loop peeling.

diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index 753224dd367b36846fac3cfc380c8453879eddf1..b86e17d1a95c737bd12950c12b6f5f7bdbd273dd 100644 (file)
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -2595,7 +2595,8 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
       || loop->inner
       /* We don't currently maintaing the LCSSA for prologue peeled inversed
         loops.  */
-      || LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
+      || (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)
+         && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)))
     do_peeling = false;
 
   struct _vect_peel_extended_info peel_for_known_alignment;

diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index f859f2c57113d4823c9f036f3e8e20c1f5dcde27..5d7d599b9749060063ca324bed21d59e2f158541 100644 (file)
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -1482,7 +1482,7 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
                                        edge scalar_exit, edge e, edge *new_e,
                                        bool flow_loops,
                                        vec<basic_block> *updated_doms,
-                                       bool uncounted_p)
+                                       bool uncounted_p, bool create_main_e)
 {
   class loop *new_loop;
   basic_block *new_bbs, *bbs, *pbbs;
@@ -1940,10 +1940,60 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
       flush_pending_stmts (entry_e);
       set_immediate_dominator (CDI_DOMINATORS, new_preheader, entry_e->src);
 
+
+      /* `vect_set_loop_condition' replaces the condition in the main exit of
+        loop.  For counted loops, this is the IV counting exit, so in the case
+        of the prolog loop, we are replacing the old IV counting exit limit of
+        total loop niters for the new limit of the prolog niters, as desired.
+        For uncounted loops, we don't have an IV-counting exit to replace, so
+        we add a dummy exit to be consumed by `vect_set_loop_condition' later
+        on.  */
+      if (create_main_e)
+       {
+         edge to_latch_e = single_pred_edge (new_loop->latch);
+         bool latch_is_false = to_latch_e->flags & EDGE_FALSE_VALUE ? true
+                                                                    : false;
+
+         /* Add new bb for duplicate exit.  */
+         basic_block bbcond = split_edge (to_latch_e);
+         gimple_stmt_iterator a = gsi_last_bb (bbcond);
+
+         /* Fix flags for the edge leading to the latch.  */
+         to_latch_e = find_edge (bbcond, new_loop->latch);
+         to_latch_e->flags &= ~EDGE_FALLTHRU;
+         to_latch_e->flags |= latch_is_false ? EDGE_FALSE_VALUE
+                                             : EDGE_TRUE_VALUE;
+
+         /* Build the condition.  */
+         tree cone = build_int_cst (sizetype, 1);
+         tree czero = build_int_cst (sizetype, 0);
+         gcond *cond_copy = gimple_build_cond (NE_EXPR, cone, czero, NULL_TREE,
+                                               NULL_TREE);
+
+         gsi_insert_after (&a, cond_copy, GSI_NEW_STMT);
+
+         /* Add edge for exiting the loop via new condition.  */
+         edge dup_exit = make_edge (bbcond, new_exit->dest, latch_is_false
+                               ? EDGE_TRUE_VALUE : EDGE_FALSE_VALUE);
+
+         profile_probability probability (0.5, GUESSED);
+         to_latch_e->probability = dup_exit->probability = probability;
+
+         set_immediate_dominator (CDI_DOMINATORS, dup_exit->src,
+                                  new_exit->src);
+         new_exit = dup_exit;
+         *new_e = new_exit;
+       }
+
       redirect_edge_and_branch_force (new_exit, preheader);
       flush_pending_stmts (new_exit);
       set_immediate_dominator (CDI_DOMINATORS, preheader, new_exit->src);
 
+      if (create_main_e)
+       set_immediate_dominator (CDI_DOMINATORS, scalar_exit->dest,
+                                recompute_dominator (CDI_DOMINATORS,
+                                                     scalar_exit->dest));
+
       /* And remove the non-necessary forwarder again.  Keep the other
          one so we have a proper pre-header for the loop at the exit edge.  */
       redirect_edge_pred (single_succ_edge (new_preheader),
@@ -1953,11 +2003,11 @@ slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit,
                               loop_preheader_edge (new_loop)->src);
 
       /* Update dominators for multiple exits.  */
-      if (multiple_exits_p)
+      if (multiple_exits_p || create_main_e)
        {
          for (edge alt_e : loop_exits)
            {
-             if (alt_e == loop_exit)
+             if ((alt_e == loop_exit) && !create_main_e)
                continue;
              basic_block old_dom
                = get_immediate_dominator (CDI_DOMINATORS, alt_e->dest);
@@ -3396,14 +3446,17 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
       e = loop_preheader_edge (loop);
       edge exit_e = LOOP_VINFO_MAIN_EXIT (loop_vinfo);
       gcc_checking_assert (slpeel_can_duplicate_loop_p (loop, exit_e, e)
-                          && !LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo));
+                          && (!LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)
+                              || uncounted_p));
 
       /* Peel prolog and put it on preheader edge of loop.  */
       edge scalar_e = LOOP_VINFO_SCALAR_MAIN_EXIT (loop_vinfo);
       edge prolog_e = NULL;
       prolog = slpeel_tree_duplicate_loop_to_edge_cfg (loop, exit_e,
                                                       scalar_loop, scalar_e,
-                                                      e, &prolog_e);
+                                                      e, &prolog_e, true, NULL,
+                                                      false, uncounted_p);
+
       gcc_assert (prolog);
       prolog->force_vectorize = false;
 
@@ -3456,12 +3509,15 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
 
       /* Update init address of DRs.  */
       vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR);
-      /* Update niters for vector loop.  */
-      LOOP_VINFO_NITERS (loop_vinfo)
-       = fold_build2 (MINUS_EXPR, type, niters, niters_prolog);
-      LOOP_VINFO_NITERSM1 (loop_vinfo)
-       = fold_build2 (MINUS_EXPR, type,
-                      LOOP_VINFO_NITERSM1 (loop_vinfo), niters_prolog);
+      if (!uncounted_p)
+       {
+         /* Update niters for vector loop.  */
+         LOOP_VINFO_NITERS (loop_vinfo)
+           = fold_build2 (MINUS_EXPR, type, niters, niters_prolog);
+         LOOP_VINFO_NITERSM1 (loop_vinfo)
+           = fold_build2 (MINUS_EXPR, type,
+                          LOOP_VINFO_NITERSM1 (loop_vinfo), niters_prolog);
+       }
       bool new_var_p = false;
       niters = vect_build_loop_niters (loop_vinfo, &new_var_p);
       /* It's guaranteed that vector loop bound before vectorization is at

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 2eb022e505dc555ec2f113322b3c8cdc148e48d4..8f7483297ea8d578c465c8bc4be0186ea81ba333 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -2468,7 +2468,7 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *, edge,
                                                    class loop *, edge,
                                                    edge, edge *, bool = true,
                                                    vec<basic_block> * = NULL,
-                                                   bool = false);
+                                                   bool = false, bool = false);
 class loop *vect_loop_versioning (loop_vec_info, gimple *);
 extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
                                    tree *, tree *, tree *, int, bool, bool,