Refactor load/store costing

This passes down the already available alignment scheme and
misalignment to the load/store costing routines, removing
redundant queries.

2021-10-19  Richard Biener  <rguenther@suse.de>

	* tree-vectorizer.h (vect_get_store_cost): Adjust signature.
	(vect_get_load_cost): Likewise.
	* tree-vect-data-refs.c (vect_get_data_access_cost): Get
	alignment support scheme and misalignment as arguments
	and pass them down.
	(vect_get_peeling_costs_all_drs): Compute that info here
	and note that we shouldn't need to.
	* tree-vect-stmts.c (vect_model_store_cost): Get
	alignment support scheme and misalignment as arguments.
	(vect_get_store_cost): Likewise.
	(vect_model_load_cost): Likewise.
	(vect_get_load_cost): Likewise.
	(vectorizable_store): Pass down alignment support scheme
	and misalignment to costing.
	(vectorizable_load): Likewise.

diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c
index 7c95f9ad69e0c51d1ae21b311878efaf23bbdcd8..0db6aec7312ee307383549d7db267fbcdbf548a1 100644 (file)
--- a/gcc/tree-vect-data-refs.c
+++ b/gcc/tree-vect-data-refs.c
@@ -1396,7 +1396,9 @@ vector_alignment_reachable_p (dr_vec_info *dr_info)
 
 static void
 vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
-                           unsigned int *inside_cost,
+                          dr_alignment_support alignment_support_scheme,
+                          int misalignment,
+                          unsigned int *inside_cost,
                            unsigned int *outside_cost,
                           stmt_vector_for_cost *body_cost_vec,
                           stmt_vector_for_cost *prologue_cost_vec)
@@ -1411,10 +1413,12 @@ vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
     ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
 
   if (DR_IS_READ (dr_info->dr))
-    vect_get_load_cost (vinfo, stmt_info, ncopies, true, inside_cost,
+    vect_get_load_cost (vinfo, stmt_info, ncopies, alignment_support_scheme,
+                       misalignment, true, inside_cost,
                        outside_cost, prologue_cost_vec, body_cost_vec, false);
   else
-    vect_get_store_cost (vinfo,stmt_info, ncopies, inside_cost, body_cost_vec);
+    vect_get_store_cost (vinfo,stmt_info, ncopies, alignment_support_scheme,
+                        misalignment, inside_cost, body_cost_vec);
 
   if (dump_enabled_p ())
     dump_printf_loc (MSG_NOTE, vect_location,
@@ -1545,7 +1549,15 @@ vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
                             vect_dr_misalign_for_aligned_access (dr0_info));
       else
        vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
-      vect_get_data_access_cost (loop_vinfo, dr_info, inside_cost, outside_cost,
+      /* ???  We should be able to avoid both the adjustment before and the
+        call to vect_supportable_dr_alignment below.  */
+      tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
+      int misalignment = dr_misalignment (dr_info, vectype);
+      dr_alignment_support alignment_support_scheme
+       = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype);
+      vect_get_data_access_cost (loop_vinfo, dr_info,
+                                alignment_support_scheme, misalignment,
+                                inside_cost, outside_cost,
                                 body_cost_vec, prologue_cost_vec);
       SET_DR_MISALIGNMENT (dr_info, save_misalignment);
     }

diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c
index 09a97b44c914fcb5d9f1afcd9215a01067fbccc9..afc3ef17834d4a46c0ce716a4e6a769154bd2f87 100644 (file)
--- a/gcc/tree-vect-stmts.c
+++ b/gcc/tree-vect-stmts.c
@@ -909,6 +909,8 @@ cfun_returns (tree decl)
 static void
 vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
                       vect_memory_access_type memory_access_type,
+                      dr_alignment_support alignment_support_scheme,
+                      int misalignment,
                       vec_load_store_type vls_type, slp_tree slp_node,
                       stmt_vector_for_cost *cost_vec)
 {
@@ -969,7 +971,8 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
                                       scalar_store, stmt_info, 0, vect_body);
     }
   else
-    vect_get_store_cost (vinfo, stmt_info, ncopies, &inside_cost, cost_vec);
+    vect_get_store_cost (vinfo, stmt_info, ncopies, alignment_support_scheme,
+                        misalignment, &inside_cost, cost_vec);
 
   if (memory_access_type == VMAT_ELEMENTWISE
       || memory_access_type == VMAT_STRIDED_SLP)
@@ -1021,15 +1024,12 @@ vect_model_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
 
 /* Calculate cost of DR's memory access.  */
 void
-vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
+vect_get_store_cost (vec_info *, stmt_vec_info stmt_info, int ncopies,
+                    dr_alignment_support alignment_support_scheme,
+                    int misalignment,
                     unsigned int *inside_cost,
                     stmt_vector_for_cost *body_cost_vec)
 {
-  dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  dr_alignment_support alignment_support_scheme
-    = vect_supportable_dr_alignment (vinfo, dr_info, vectype);
-
   switch (alignment_support_scheme)
     {
     case dr_aligned:
@@ -1049,8 +1049,7 @@ vect_get_store_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
         /* Here, we assign an additional cost for the unaligned store.  */
        *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
                                          unaligned_store, stmt_info,
-                                         dr_misalignment (dr_info, vectype),
-                                         vect_body);
+                                         misalignment, vect_body);
         if (dump_enabled_p ())
           dump_printf_loc (MSG_NOTE, vect_location,
                            "vect_model_store_cost: unaligned supported by "
@@ -1085,6 +1084,8 @@ static void
 vect_model_load_cost (vec_info *vinfo,
                      stmt_vec_info stmt_info, unsigned ncopies, poly_uint64 vf,
                      vect_memory_access_type memory_access_type,
+                     dr_alignment_support alignment_support_scheme,
+                     int misalignment,
                      gather_scatter_info *gs_info,
                      slp_tree slp_node,
                      stmt_vector_for_cost *cost_vec)
@@ -1144,7 +1145,8 @@ vect_model_load_cost (vec_info *vinfo,
            dump_printf_loc (MSG_NOTE, vect_location,
                             "vect_model_load_cost: %d unused vectors.\n",
                             gaps);
-         vect_get_load_cost (vinfo, stmt_info, ncopies * gaps, false,
+         vect_get_load_cost (vinfo, stmt_info, ncopies * gaps,
+                             alignment_support_scheme, misalignment, false,
                              &inside_cost, &prologue_cost,
                              cost_vec, cost_vec, true);
        }
@@ -1190,7 +1192,8 @@ vect_model_load_cost (vec_info *vinfo,
                                       scalar_load, stmt_info, 0, vect_body);
     }
   else
-    vect_get_load_cost (vinfo, stmt_info, ncopies, first_stmt_p,
+    vect_get_load_cost (vinfo, stmt_info, ncopies,
+                       alignment_support_scheme, misalignment, first_stmt_p,
                        &inside_cost, &prologue_cost, 
                        cost_vec, cost_vec, true);
   if (memory_access_type == VMAT_ELEMENTWISE
@@ -1209,18 +1212,15 @@ vect_model_load_cost (vec_info *vinfo,
 
 /* Calculate cost of DR's memory access.  */
 void
-vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
+vect_get_load_cost (vec_info *, stmt_vec_info stmt_info, int ncopies,
+                   dr_alignment_support alignment_support_scheme,
+                   int misalignment,
                    bool add_realign_cost, unsigned int *inside_cost,
                    unsigned int *prologue_cost,
                    stmt_vector_for_cost *prologue_cost_vec,
                    stmt_vector_for_cost *body_cost_vec,
                    bool record_prologue_costs)
 {
-  dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
-  tree vectype = STMT_VINFO_VECTYPE (stmt_info);
-  dr_alignment_support alignment_support_scheme
-    = vect_supportable_dr_alignment (vinfo, dr_info, vectype);
-
   switch (alignment_support_scheme)
     {
     case dr_aligned:
@@ -1239,8 +1239,7 @@ vect_get_load_cost (vec_info *vinfo, stmt_vec_info stmt_info, int ncopies,
         /* Here, we assign an additional cost for the unaligned load.  */
        *inside_cost += record_stmt_cost (body_cost_vec, ncopies,
                                          unaligned_load, stmt_info,
-                                         dr_misalignment (dr_info, vectype),
-                                         vect_body);
+                                         misalignment, vect_body);
 
         if (dump_enabled_p ())
           dump_printf_loc (MSG_NOTE, vect_location,
@@ -7443,7 +7442,8 @@ vectorizable_store (vec_info *vinfo,
 
       STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
       vect_model_store_cost (vinfo, stmt_info, ncopies,
-                            memory_access_type, vls_type, slp_node, cost_vec);
+                            memory_access_type, alignment_support_scheme,
+                            misalignment, vls_type, slp_node, cost_vec);
       return true;
     }
   gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
@@ -8779,6 +8779,7 @@ vectorizable_load (vec_info *vinfo,
 
       STMT_VINFO_TYPE (stmt_info) = load_vec_info_type;
       vect_model_load_cost (vinfo, stmt_info, ncopies, vf, memory_access_type,
+                           alignment_support_scheme, misalignment,
                            &gs_info, slp_node, cost_vec);
       return true;
     }

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 954a0e18d031ee91f4f7010cb9945bde4f81702d..746e39207d04e4af05110379483258c156090f0b 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1957,11 +1957,13 @@ extern bool vect_nop_conversion_p (stmt_vec_info);
 extern opt_result vect_analyze_stmt (vec_info *, stmt_vec_info, bool *,
                                     slp_tree,
                                     slp_instance, stmt_vector_for_cost *);
-extern void vect_get_load_cost (vec_info *, stmt_vec_info, int, bool,
+extern void vect_get_load_cost (vec_info *, stmt_vec_info, int,
+                               dr_alignment_support, int, bool,
                                unsigned int *, unsigned int *,
                                stmt_vector_for_cost *,
                                stmt_vector_for_cost *, bool);
 extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
+                                dr_alignment_support, int,
                                 unsigned int *, stmt_vector_for_cost *);
 extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
 extern tree vect_gen_perm_mask_any (tree, const vec_perm_indices &);