Record gather/scatter scale and base in the SLP tree

The main gather/scatter discovery happens at SLP discovery time,
the base address and the offset scale are currently not explicitly
represented in the SLP tree.  This requires re-discovery of them
during vectorizable_store/load.  The following fixes this by
recording this info into the SLP tree.  This allows the main
vect_check_gather_scatter call to be elided from get_load_store_type
and replaced with target support checks for IFN/decl or fallback
emulated mode.

There's vect_check_gather_scatter left in the path using gather/scatter
for strided load/store.  I hope to deal with this later.

	* tree-vectorizer.h (_slp_tree::gs_scale): New.
	(_slp_tree::gs_base): Likewise.
	(SLP_TREE_GS_SCALE): Likewise.
	(SLP_TREE_GS_BASE): Likewise.
	(vect_describe_gather_scatter_call): Declare.
	* tree-vect-slp.cc (_slp_tree::_slp_tree): Initialize
	new members.
	(vect_build_slp_tree_2): Record gather/scatter base and scale.
	(vect_get_and_check_slp_defs): For gather/scatter IFNs
	describe the call to first_gs_info.
	* tree-vect-data-refs.cc (vect_gather_scatter_fn_p): Add
	mode of operation with fixed offset vector type.
	(vect_describe_gather_scatter_call): Export.
	* tree-vect-stmts.cc (get_load_store_type): Do not call
	vect_check_gather_scatter to fill gs_info, instead populate
	from the SLP tree.  Check which of, IFN, decl or fallback
	is supported and record that decision.

diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index dc82bf688abaf6e9cc0902e8de36200fd1947e53..4db0ec8abae6e6641c7c021e05a5b69ca9c5e01b 100644 (file)
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -4423,8 +4423,9 @@ vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
    MASKED_P is true if the load or store is conditional.  MEMORY_TYPE is
    the type of the memory elements being loaded or stored.  OFFSET_TYPE
    is the type of the offset that is being applied to the invariant
-   base address.  SCALE is the amount by which the offset should
-   be multiplied *after* it has been converted to address width.
+   base address.  If OFFSET_TYPE is scalar the function chooses an
+   appropriate vector type for it.  SCALE is the amount by which the
+   offset should be multiplied *after* it has been converted to address width.
 
    Return true if the function is supported, storing the function id in
    *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.
@@ -4467,9 +4468,15 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
 
   for (;;)
     {
-      tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
-      if (!offset_vectype)
-       return false;
+      tree offset_vectype;
+      if (VECTOR_TYPE_P (offset_type))
+       offset_vectype = offset_type;
+      else
+       {
+         offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
+         if (!offset_vectype)
+           return false;
+       }
 
       /* Test whether the target supports this combination.  */
       if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
@@ -4500,10 +4507,15 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
          return true;
        }
 
+      /* For fixed offset vector type we're done.  */
+      if (VECTOR_TYPE_P (offset_type))
+       return false;
+
       if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
          && TYPE_PRECISION (offset_type) >= element_bits)
        return false;
 
+      /* Try a larger offset vector type.  */
       offset_type = build_nonstandard_integer_type
        (TYPE_PRECISION (offset_type) * 2, TYPE_UNSIGNED (offset_type));
     }
@@ -4512,7 +4524,7 @@ vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
 /* STMT_INFO is a call to an internal gather load or scatter store function.
    Describe the operation in INFO.  */
 
-static void
+void
 vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
                                   gather_scatter_info *info)
 {

diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 794a073b3758d1efe3f48b0094ff5178ce124241..59acbc1aeb571c873c6c681bf6717d1eaebd5f4f 100644 (file)
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -120,6 +120,8 @@ _slp_tree::_slp_tree ()
   SLP_TREE_LANE_PERMUTATION (this) = vNULL;
   SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
   SLP_TREE_CODE (this) = ERROR_MARK;
+  SLP_TREE_GS_SCALE (this) = 0;
+  SLP_TREE_GS_BASE (this) = NULL_TREE;
   this->ldst_lanes = false;
   this->avoid_stlf_fail = false;
   SLP_TREE_VECTYPE (this) = NULL_TREE;
@@ -680,6 +682,15 @@ vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap,
        {
          internal_fn ifn = gimple_call_internal_fn (stmt);
          commutative_op = first_commutative_argument (ifn);
+         if (internal_gather_scatter_fn_p (ifn))
+           {
+             vect_describe_gather_scatter_call
+               (stmt_info,
+                first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
+             if (first)
+               (*oprnds_info)[0]->first_gs_p = true;
+             gs_op = 0;
+           }
        }
     }
   else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
@@ -2720,6 +2731,9 @@ out:
 
   stmt_info = stmts[0];
 
+  int gs_scale = 0;
+  tree gs_base = NULL_TREE;
+
   /* Create SLP_TREE nodes for the definition node/s.  */
   FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
     {
@@ -2742,6 +2756,12 @@ out:
          continue;
        }
 
+      if (oprnd_info->first_gs_p)
+       {
+         gs_scale = oprnd_info->first_gs_info.scale;
+         gs_base = oprnd_info->first_gs_info.base;
+       }
+
       if (is_a <bb_vec_info> (vinfo)
          && oprnd_info->first_dt == vect_internal_def
          && !oprnd_info->any_pattern)
@@ -3131,6 +3151,8 @@ fail:
   node = vect_create_new_slp_node (node, stmts, nops);
   SLP_TREE_VECTYPE (node) = vectype;
   SLP_TREE_CHILDREN (node).splice (children);
+  SLP_TREE_GS_SCALE (node) = gs_scale;
+  SLP_TREE_GS_BASE (node) = gs_base;
   return node;
 }
 

diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 3bf2deca8eaae342447bfde0dec5aeb2dc278253..a6f4db43a495bf728bb77107ad70d9178b6f4eac 100644 (file)
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -2019,31 +2019,40 @@ get_load_store_type (vec_info  *vinfo, stmt_vec_info stmt_info,
   else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
     {
       *memory_access_type = VMAT_GATHER_SCATTER;
-      if (!vect_check_gather_scatter (stmt_info, loop_vinfo, gs_info,
-                                     elsvals))
-       gcc_unreachable ();
       slp_tree offset_node = SLP_TREE_CHILDREN (slp_node)[0];
       tree offset_vectype = SLP_TREE_VECTYPE (offset_node);
+      memset (gs_info, 0, sizeof (gather_scatter_info));
       gs_info->offset_vectype = offset_vectype;
-      /* When using internal functions, we rely on pattern recognition
-        to convert the type of the offset to the type that the target
-        requires, with the result being a call to an internal function.
-        If that failed for some reason (e.g. because another pattern
-        took priority), just handle cases in which the offset already
-        has the right type.  */
-      if (GATHER_SCATTER_IFN_P (*gs_info)
-         && !is_gimple_call (stmt_info->stmt)
-         && !tree_nop_conversion_p (TREE_TYPE (gs_info->offset),
-                                    TREE_TYPE (offset_vectype)))
-       {
-         if (dump_enabled_p ())
-           dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                            "%s offset requires a conversion\n",
-                            vls_type == VLS_LOAD ? "gather" : "scatter");
-         return false;
-       }
-      else if (GATHER_SCATTER_EMULATED_P (*gs_info))
+      gs_info->scale = SLP_TREE_GS_SCALE (slp_node);
+      gs_info->base = SLP_TREE_GS_BASE (slp_node);
+      gs_info->memory_type = TREE_TYPE (DR_REF (first_dr_info->dr));
+      gs_info->decl = NULL_TREE;
+      gs_info->ifn = IFN_LAST;
+      tree tem;
+      if (vect_gather_scatter_fn_p (loop_vinfo, vls_type == VLS_LOAD,
+                                   masked_p, vectype,
+                                   gs_info->memory_type,
+                                   offset_vectype, gs_info->scale,
+                                   &gs_info->ifn, &tem,
+                                   elsvals))
+       /* GATHER_SCATTER_IFN_P.  */;
+      else if (vls_type == VLS_LOAD
+              ? (targetm.vectorize.builtin_gather
+                 && (gs_info->decl
+                       = targetm.vectorize.builtin_gather (vectype,
+                                                           TREE_TYPE
+                                                             (offset_vectype),
+                                                           gs_info->scale)))
+              : (targetm.vectorize.builtin_scatter
+                 && (gs_info->decl
+                       = targetm.vectorize.builtin_scatter (vectype,
+                                                            TREE_TYPE
+                                                              (offset_vectype),
+                                                            gs_info->scale))))
+       /* GATHER_SCATTER_LEGACY_P.  */;
+      else
        {
+         /* GATHER_SCATTER_EMULATED_P.  */
          if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant ()
              || !TYPE_VECTOR_SUBPARTS (offset_vectype).is_constant ()
              || VECTOR_BOOLEAN_TYPE_P (offset_vectype)

diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 52e1075f7e49ab027cfb37177f352659c4112fa3..9653496362ff82e96b73f037a636b24e01ee9a57 100644 (file)
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -306,6 +306,11 @@ struct _slp_tree {
   unsigned int lanes;
   /* The operation of this node.  */
   enum tree_code code;
+  /* For gather/scatter memory operations the scale each offset element
+     should be multiplied by before being added to the base.  */
+  int gs_scale;
+  /* For gather/scatter memory operations the loop-invariant base value.  */
+  tree gs_base;
   /* Whether uses of this load or feeders of this store are suitable
      for load/store-lanes.  */
   bool ldst_lanes;
@@ -412,6 +417,8 @@ public:
 #define SLP_TREE_CODE(S)                        (S)->code
 #define SLP_TREE_MEMORY_ACCESS_TYPE(S)          (S)->memory_access_type
 #define SLP_TREE_TYPE(S)                        (S)->type
+#define SLP_TREE_GS_SCALE(S)                    (S)->gs_scale
+#define SLP_TREE_GS_BASE(S)                     (S)->gs_base
 
 enum vect_partial_vector_style {
     vect_partial_vectors_none,
@@ -2550,6 +2557,8 @@ extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
 extern bool vect_check_gather_scatter (stmt_vec_info, loop_vec_info,
                                       gather_scatter_info *,
                                       vec<int> * = nullptr);
+extern void vect_describe_gather_scatter_call (stmt_vec_info,
+                                              gather_scatter_info *);
 extern opt_result vect_find_stmt_data_reference (loop_p, gimple *,
                                                 vec<data_reference_p> *,
                                                 vec<int> *, int);