i386: Prefer shorter fold-left reduction chains

Fold-left FP reductions are lowered to scalar operations on x86.  In
mixed-width loops the per-statement costs can therefore make a wider
vector mode look better even though it lengthens the serial reduction
chain.

Count the fold-left reduction lanes from reduction groups and chains during
finish_cost and use that as an extra loop-candidate preference.  Do not let
emulated sub-SSE modes win on this preference alone.  Dump the lane count when
present.

gcc/ChangeLog:

	* config/i386/i386.cc (ix86_vector_costs): Add
	better_fold_left_reduc_than_p and m_num_fold_left_reduc_lanes.
	(ix86_vector_costs::ix86_vector_costs): Initialize it.
	(ix86_vector_costs::finish_cost): Count and dump fold-left
	reduction lanes.
	(ix86_vector_costs::better_fold_left_reduc_than_p): New function.
	(ix86_vector_costs::better_main_loop_than_p): Use it.
	(ix86_vector_costs::better_epilogue_loop_than_p): Likewise.

gcc/testsuite/ChangeLog:

	* gcc.target/i386/fold-left-reduc-cost.c: New test.
	* gcc.target/i386/fold-left-reduc-chain-cost.c: New test.
	* gcc.target/i386/vect-epilogues-6.c: Update expected vector size.
	* gcc.target/i386/vect-epilogues-7.c: Likewise.
	* gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c: Likewise.

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 804f35b29739c312b1927e0718c234c2c97c758a..33e5e063b991fe589f574527657d77f0b47bdc67 100644 (file)
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -26452,6 +26452,7 @@ public:
 
 private:
 
+  bool better_fold_left_reduc_than_p (const vector_costs *) const;
   /* Estimate register pressure of the vectorized code.  */
   void ix86_vect_estimate_reg_pressure ();
   /* Number of GENERAL_REGS/SSE_REGS used in the vectorizer, it's used for
@@ -26468,6 +26469,8 @@ private:
   unsigned m_num_reduc[X86_REDUC_LAST];
   /* Don't do unroll if m_prefer_unroll is false, default is true.  */
   bool m_prefer_unroll;
+  /* Scalar lanes in fold-left reductions.  */
+  unsigned int m_num_fold_left_reduc_lanes;
 };
 
 ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
@@ -26477,7 +26480,8 @@ ix86_vector_costs::ix86_vector_costs (vec_info* vinfo, bool costing_for_scalar)
     m_num_avx256_vec_perm (),
     m_num_avx512_vec_perm (),
     m_num_reduc (),
-    m_prefer_unroll (true)
+    m_prefer_unroll (true),
+    m_num_fold_left_reduc_lanes (0)
 {}
 
 /* Implement targetm.vectorize.create_costs.  */
@@ -27183,6 +27187,20 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
   loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo);
   if (loop_vinfo && !m_costing_for_scalar)
     {
+      unsigned int vf = vect_vf_for_cost (loop_vinfo);
+      for (auto inst : LOOP_VINFO_SLP_INSTANCES (loop_vinfo))
+       if ((SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_group
+            || SLP_INSTANCE_KIND (inst) == slp_inst_kind_reduc_chain)
+           && (vect_reduc_type (loop_vinfo, SLP_INSTANCE_TREE (inst))
+               == FOLD_LEFT_REDUCTION))
+         m_num_fold_left_reduc_lanes
+           += vf * SLP_TREE_LANES (SLP_INSTANCE_TREE (inst));
+
+      if (m_num_fold_left_reduc_lanes && dump_enabled_p ())
+       dump_printf_loc (MSG_NOTE, vect_location,
+                        "in-order FP reduction lanes: %u\n",
+                        m_num_fold_left_reduc_lanes);
+
       /* We are currently not asking the vectorizer to compare costs
         between different vector mode sizes.  When using predication
         that will end up always choosing the preferred mode size even
@@ -27347,6 +27365,21 @@ ix86_vector_costs::finish_cost (const vector_costs *scalar_costs)
   vector_costs::finish_cost (scalar_costs);
 }
 
+/* Return true if THIS has a shorter fold-left reduction chain than OTHER.  */
+
+bool
+ix86_vector_costs::better_fold_left_reduc_than_p
+  (const vector_costs *other) const
+{
+  auto other_costs = static_cast<const ix86_vector_costs *> (other);
+  if (m_num_fold_left_reduc_lanes >= other_costs->m_num_fold_left_reduc_lanes)
+    return false;
+
+  /* Do not let emulated sub-SSE modes win on this alone.  */
+  loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
+  return known_ge (GET_MODE_SIZE (loop_vinfo->vector_mode), 16);
+}
+
 /* Return true if THIS should be preferred over OTHER as main vector loop.  */
 
 bool
@@ -27355,6 +27388,9 @@ ix86_vector_costs::better_main_loop_than_p (const vector_costs *other) const
   loop_vec_info this_loop_vinfo = as_a<loop_vec_info> (this->vinfo ());
   loop_vec_info other_loop_vinfo = as_a<loop_vec_info> (other->vinfo ());
 
+  if (better_fold_left_reduc_than_p (other))
+    return true;
+
   /* If the other loop is masked it does not need an epilog.  Prefer that
      if the current loop cannot be vectorized fully with a vector
      epilogs with at most one scalar iteration left.  */
@@ -27377,6 +27413,10 @@ ix86_vector_costs::better_epilogue_loop_than_p (const vector_costs *other,
                                                loop_vec_info main_loop) const
 {
   loop_vec_info this_loop_info = as_a <loop_vec_info> (this->vinfo ());
+
+  if (better_fold_left_reduc_than_p (other))
+    return true;
+
   /* The x86 target allows for multiple vector epilogues, if THIS is
      the suggested epilog mode of OTHER then keep the latter unless
      THIS has a VF of one which means no further epilog needed.  */

diff --git a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
index abb9f6681c2eff8a95537f32a1007847bb4c1323..370e7ed4d11e49687c58ddddcfa5f83887edd3df 100644 (file)
--- a/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
+++ b/gcc/testsuite/gcc.dg/vect/costmodel/x86_64/costmodel-vect-epil-1.c
@@ -52,7 +52,6 @@ void test (const unsigned char * __restrict__ pi,
     pp_avg_rgb[3] = pp_avg_rgb_3;
 }
 
-/* Even though there's an SLP opportunity in-order reductions should never use
-   masked epilogs.  */
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */
+/* In-order reductions should not use masked epilogs here.  */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 16 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump-not "optimized: epilogue loop vectorized using masked" "vect" } } */

diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
new file mode 100644 (file)
index 0000000..6346472
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-chain-cost.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The two dependent updates form a fold-left reduction chain.  */
+
+float
+foo (float *__restrict__ a, int n)
+{
+  float sum = 0;
+  for (int i = 0; i != n; i++)
+    {
+      sum += a[2 * i];
+      sum += a[2 * i + 1];
+    }
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "Starting SLP discovery of reduction chain" "vect" } } */
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */

diff --git a/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
new file mode 100644 (file)
index 0000000..6babd76
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/fold-left-reduc-cost.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -march=x86-64-v3 -fdump-tree-vect-details" } */
+
+/* The byte loads make V32QI available, but the fold-left reduction should
+   make the smaller SSE loop win.  */
+
+float
+foo (char *a, char *b, int n)
+{
+  float sum = 0;
+  for (int i = 0; i != n; i++)
+    sum += a[i] * b[i];
+  return sum;
+}
+
+/* { dg-final { scan-tree-dump "in-order FP reduction lanes" "vect" } } */
+/* { dg-final { scan-tree-dump "loop vectorized using 16 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump-not "loop vectorized using 32 byte vectors" "vect" } } */

diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
index 8cd8740c6ecc5cabd49559424d952df66dcad8e1..9f0445390f4fee4c8dcc37f5dabdab70890264d5 100644 (file)
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-6.c
@@ -17,5 +17,4 @@ foo (double *a, char *mask, int n)
   return sum;
 }
 
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using 32 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */

diff --git a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
index 63c29895f9bb69d449b3113c8c22a998ad0a3243..be56819fc54772a5165e0d0189274d674b3e3c16 100644 (file)
--- a/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
+++ b/gcc/testsuite/gcc.target/i386/vect-epilogues-7.c
@@ -17,5 +17,5 @@ foo (double *a, char *mask, int n)
   return sum;
 }
 
-/* { dg-final { scan-tree-dump "optimized: loop vectorized using 64 byte vectors" "vect" } } */
-/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 64 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: loop vectorized using 32 byte vectors" "vect" } } */
+/* { dg-final { scan-tree-dump "optimized: epilogue loop vectorized using masked 32 byte vectors" "vect" } } */