Add LOOP_VINFO_DRS_ADVANCED_BY

The following remembers how we advanced DRs when vectorizing an
epilogue.  When we want to vectorize the epilogue of such epilogue
we have to retain that advancement and add the advancement for this
vectorized epilogue.  Due to the way we copy and re-associate
stmt_vec_infos and DRs recording this advancement and re-applying
it for the next epilogue is simplest.

	* tree-vectorizer.h (_loop_vec_info::drs_advanced_by): New.
	(LOOP_VINFO_DRS_ADVANCED_BY): Likewise.
	* tree-vect-loop.cc (_loop_vec_info::_loop_vec_info): Initialize
	drs_advanced_by.
	(update_epilogue_loop_vinfo): Remember the DR advancement made.
	(vect_transform_loop): Accumulate past advancements.

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index e91549a643b94d394ea54d15afb99ced50b65d4a..6059ce031d1415d8d58f0818a873d0dd4c2fe996 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -1072,6 +1072,7 @@ _loop_vec_info::_loop_vec_info (class loop *loop_in, vec_info_shared *shared)
     scalar_loop_scaling (profile_probability::uninitialized ()),
     scalar_loop (NULL),
     orig_loop_info (NULL),
+    drs_advanced_by (NULL_TREE),
     vec_loop_iv_exit (NULL),
     vec_epilogue_loop_iv_exit (NULL),
     scalar_loop_iv_exit (NULL)
@@ -12302,6 +12303,9 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
      loop and its prologue.  */
   vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
 
+  /* Remember the advancement made.  */
+  LOOP_VINFO_DRS_ADVANCED_BY (epilogue_vinfo) = advance;
+
   epilogue_vinfo->shared->datarefs_copy.release ();
   epilogue_vinfo->shared->save_datarefs ();
 }
@@ -12849,6 +12853,11 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
 
   if (epilogue)
     {
+      /* Accumulate past advancements made.  */
+      if (LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo))
+       advance = fold_build2 (PLUS_EXPR, TREE_TYPE (advance),
+                              LOOP_VINFO_DRS_ADVANCED_BY (loop_vinfo),
+                              advance);
       update_epilogue_loop_vinfo (epilogue, advance);
 
       epilogue->simduid = loop->simduid;

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 24227a69d4acef1404ca4cfd3536377afe50164b..5a1bd237beb6a228da1fa37f4f44ee43a238a1f7 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -994,6 +994,9 @@ public:
      analysis.  */
   vec<_loop_vec_info *> epilogue_vinfos;
 
+  /* If this is an epilogue loop the DR advancement applied.  */
+  tree drs_advanced_by;
+
   /* The controlling loop IV for the current loop when vectorizing.  This IV
      controls the natural exits of the loop.  */
   edge vec_loop_iv_exit;
@@ -1097,6 +1100,7 @@ public:
 #define LOOP_VINFO_SIMD_IF_COND(L)         (L)->simd_if_cond
 #define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
 #define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L)  (L)->inv_pattern_def_seq
+#define LOOP_VINFO_DRS_ADVANCED_BY(L)      (L)->drs_advanced_by
 
 #define LOOP_VINFO_FULLY_MASKED_P(L)           \
   (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L)      \