Delay setting of slp_node->data in vectorizable_{load,store}

We move 'ls' to slp_node->data early, inhibiting late adjustments
like setting of ls_type.  We also have failure cases after, which
we should not.  The following moves one such failure check earlier
and moves setting slp_node->data and SLP_TREE_TYPE
down, duplicating it to various "success" returns.

	* tree-vect-stmts.cc (vectorizable_store): Set slp_node->data
	and SLP_TREE_TYPE only on success.
	(vectorizable_load): Likewise.  Move one validity check early.

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index a74e03cc0f65b4e436fb4a5da6ca1d9d3a6e189c..05b141bde1a646a15d8d1ea0f5ed7507b2516982 100644 (file)
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -8330,9 +8330,6 @@ vectorizable_store (vec_info *vinfo,
          && alignment_support_scheme != dr_aligned)
        dump_printf_loc (MSG_NOTE, vect_location,
                         "Vectorizing an unaligned access.\n");
-
-      SLP_TREE_TYPE (slp_node) = store_vec_info_type;
-      slp_node->data = new vect_load_store_data (std::move (ls));
     }
 
   /* Transform.  */
@@ -8359,6 +8356,9 @@ vectorizable_store (vec_info *vinfo,
                             "prologue_cost = %d .\n",
                             inside_cost, prologue_cost);
 
+         SLP_TREE_TYPE (slp_node) = store_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
+
          return true;
        }
       return vectorizable_scan_store (vinfo, stmt_info, slp_node, gsi);
@@ -8655,6 +8655,9 @@ vectorizable_store (vec_info *vinfo,
                             "vect_model_store_cost: inside_cost = %d, "
                             "prologue_cost = %d .\n",
                             inside_cost, prologue_cost);
+
+         SLP_TREE_TYPE (slp_node) = store_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
        }
 
       return true;
@@ -8887,6 +8890,9 @@ vectorizable_store (vec_info *vinfo,
                             "vect_model_store_cost: inside_cost = %d, "
                             "prologue_cost = %d .\n",
                             inside_cost, prologue_cost);
+
+         SLP_TREE_TYPE (slp_node) = store_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
        }
 
       return true;
@@ -9270,11 +9276,16 @@ vectorizable_store (vec_info *vinfo,
            }
        }
 
-      if (costing_p && dump_enabled_p ())
-       dump_printf_loc (MSG_NOTE, vect_location,
-                        "vect_model_store_cost: inside_cost = %d, "
-                        "prologue_cost = %d .\n",
-                        inside_cost, prologue_cost);
+      if (costing_p)
+       {
+         if (dump_enabled_p ())
+           dump_printf_loc (MSG_NOTE, vect_location,
+                            "vect_model_store_cost: inside_cost = %d, "
+                            "prologue_cost = %d .\n",
+                            inside_cost, prologue_cost);
+         SLP_TREE_TYPE (slp_node) = store_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
+       }
 
       return true;
     }
@@ -9539,6 +9550,9 @@ vectorizable_store (vec_info *vinfo,
                         "vect_model_store_cost: inside_cost = %d, "
                         "prologue_cost = %d .\n",
                         inside_cost, prologue_cost);
+
+      SLP_TREE_TYPE (slp_node) = store_vec_info_type;
+      slp_node->data = new vect_load_store_data (std::move (ls));
     }
 
   return true;
@@ -9917,6 +9931,26 @@ vectorizable_load (vec_info *vinfo,
 
   if (costing_p) /* transformation not required.  */
     {
+      if (loop_vinfo
+         && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
+       check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
+                                             VLS_LOAD, group_size, &ls,
+                                             mask_node, &ls.elsvals);
+
+      /* If the type needs padding we must zero inactive elements.
+        Check if we can do that with a VEC_COND_EXPR and store the
+        elsval we choose in MASKLOAD_ELSVAL.  */
+      if (ls.elsvals.length ()
+         && type_mode_padding_p
+         && !ls.elsvals.contains (MASK_LOAD_ELSE_ZERO)
+         && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
+       {
+         if (dump_enabled_p ())
+           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                            "cannot zero inactive elements.\n");
+         return false;
+       }
+
       if (mask_node
          && !vect_maybe_update_slp_op_vectype (mask_node,
                                                mask_vectype))
@@ -9927,12 +9961,6 @@ vectorizable_load (vec_info *vinfo,
          return false;
        }
 
-      if (loop_vinfo
-         && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
-       check_load_store_for_partial_vectors (loop_vinfo, vectype, slp_node,
-                                             VLS_LOAD, group_size, &ls,
-                                             mask_node, &ls.elsvals);
-
       if (dump_enabled_p ()
          && memory_access_type != VMAT_ELEMENTWISE
          && !mat_gather_scatter_p (memory_access_type)
@@ -9944,23 +9972,6 @@ vectorizable_load (vec_info *vinfo,
 
       if (memory_access_type == VMAT_LOAD_STORE_LANES)
        vinfo->any_known_not_updated_vssa = true;
-
-      SLP_TREE_TYPE (slp_node) = load_vec_info_type;
-      slp_node->data = new vect_load_store_data (std::move (ls));
-    }
-
-  /* If the type needs padding we must zero inactive elements.
-     Check if we can do that with a VEC_COND_EXPR and store the
-     elsval we choose in MASKLOAD_ELSVAL.  */
-  if (elsvals.length ()
-      && type_mode_padding_p
-      && !elsvals.contains (MASK_LOAD_ELSE_ZERO)
-      && !expand_vec_cond_expr_p (vectype, truth_type_for (vectype)))
-    {
-      if (dump_enabled_p ())
-       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                        "cannot zero inactive elements.\n");
-      return false;
     }
 
   /* For now just use the first available else value.
@@ -10046,6 +10057,8 @@ vectorizable_load (vec_info *vinfo,
                             "vect_model_load_cost: inside_cost = %d, "
                             "prologue_cost = %d .\n",
                             inside_cost, prologue_cost);
+         SLP_TREE_TYPE (slp_node) = load_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
          return true;
        }
       if (hoist_p)
@@ -10458,6 +10471,8 @@ vectorizable_load (vec_info *vinfo,
                             "vect_model_load_cost: inside_cost = %u, "
                             "prologue_cost = 0 .\n",
                             inside_cost);
+         SLP_TREE_TYPE (slp_node) = load_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
        }
 
       return true;
@@ -10848,6 +10863,8 @@ vectorizable_load (vec_info *vinfo,
                             "vect_model_load_cost: inside_cost = %u, "
                             "prologue_cost = %u .\n",
                             inside_cost, prologue_cost);
+         SLP_TREE_TYPE (slp_node) = load_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
        }
 
       return true;
@@ -11281,11 +11298,16 @@ vectorizable_load (vec_info *vinfo,
            }
        }
 
-      if (costing_p && dump_enabled_p ())
-       dump_printf_loc (MSG_NOTE, vect_location,
-                        "vect_model_load_cost: inside_cost = %u, "
-                        "prologue_cost = %u .\n",
-                        inside_cost, prologue_cost);
+      if (costing_p)
+       {
+         if (dump_enabled_p ())
+           dump_printf_loc (MSG_NOTE, vect_location,
+                            "vect_model_load_cost: inside_cost = %u, "
+                            "prologue_cost = %u .\n",
+                            inside_cost, prologue_cost);
+         SLP_TREE_TYPE (slp_node) = load_vec_info_type;
+         slp_node->data = new vect_load_store_data (std::move (ls));
+       }
       return true;
     }
 
@@ -11948,6 +11970,8 @@ vectorizable_load (vec_info *vinfo,
                         "vect_model_load_cost: inside_cost = %u, "
                         "prologue_cost = %u .\n",
                         inside_cost, prologue_cost);
+      SLP_TREE_TYPE (slp_node) = load_vec_info_type;
+      slp_node->data = new vect_load_store_data (std::move (ls));
     }
 
   return true;