tree-optimization/121256 - properly support SLP in vectorizable recurrence

We failed to build the correct initialization vector.  For VLA
vectors and a non-uniform initialization vector this rejects
vectorization for now.

	PR tree-optimization/121256
	* tree-vect-loop.cc (vectorizable_recurr): Build a correct
	initialization vector for SLP_TREE_LANES > 1.

	* gcc.dg/vect/vect-recurr-pr121256.c: New testcase.
	* gcc.dg/vect/vect-recurr-pr121256-2.c: Likewise.

(cherry picked from commit 26dc9aa285b53551c55d3d660bb6da21d59d7023)

diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
new file mode 100644 (file)
index 0000000..7350fd9
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256-2.c
@@ -0,0 +1,49 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+
+int red = 153;
+int green = 66;
+int blue = 187;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(int *restrict dst, int *restrict src)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 3 + R];
+    int gt = src[i * 3 + G];
+    int bt = src[i * 3 + B];
+
+    dst[i * 3 + R] = rt - red;
+    dst[i * 3 + G] = gt - green;
+    dst[i * 3 + B] = bt - blue;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+  }
+}
+
+int main()
+{
+  int dst[8*3];
+  int src[8*3] = { 160, 73, 194, 17, 33, 99, 0, 12, 283, 87, 73, 11,
+                  9, 7, 1, 23, 19, 13, 77, 233, 97, 78, 2, 5 };
+  int dst2[8*3] = {-27, 7, 41, -143, -40, -95, -17, -21, 184, 87, 61,
+      -272, -78, -66, -10, 14, 12, 12, 54, 214, 84, 1, -231, -92};
+
+  check_vect ();
+
+  sub_left_prediction_bgr32(dst, src);
+
+#pragma GCC novector
+  for (int i = 0; i < 8*3; ++i)
+    if (dst[i] != dst2[i])
+      __builtin_abort();
+
+  return 0;
+}

diff --git a/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
new file mode 100644 (file)
index 0000000..c895e94
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-recurr-pr121256.c
@@ -0,0 +1,54 @@
+/* { dg-additional-options "-mavx2" { target avx2_runtime } } */
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include "tree-vect.h"
+
+#define B 0
+#define G 1
+#define R 2
+#define A 3
+
+int red = 153;
+int green = 66;
+int blue = 187;
+int alpha = 255;
+
+static void __attribute__((noipa))
+sub_left_prediction_bgr32(uint8_t *restrict dst, uint8_t *restrict src, int w)
+{
+  for (int i = 0; i < 8; i++) {
+    int rt = src[i * 4 + R];
+    int gt = src[i * 4 + G];
+    int bt = src[i * 4 + B];
+    int at = src[i * 4 + A];
+
+    dst[i * 4 + R] = rt - red;
+    dst[i * 4 + G] = gt - green;
+    dst[i * 4 + B] = bt - blue;
+    dst[i * 4 + A] = at - alpha;
+
+    red = rt;
+    green = gt;
+    blue = bt;
+    alpha = at;
+  }
+}
+
+int main()
+{
+  check_vect ();
+
+  uint8_t *dst = calloc(36, sizeof(uint8_t));
+  uint8_t *src = calloc(36, sizeof(uint8_t));
+
+  src[R] = 160;
+  src[G] = 73;
+  src[B] = 194;
+  src[A] = 255;
+
+  sub_left_prediction_bgr32(dst, src, 33);
+  if (dst[R] != 7 || dst[B] != 7 || dst[A] != 0)
+    __builtin_abort();
+}

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index a38bd5723fdd0705e713cd0bc2c440bf485268aa..f22cf2ec894a58ce560c425fd13bacc30cb55bc6 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -9643,6 +9643,33 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
       return false;
     }
 
+  /* We need to be able to build a { ..., a, b } init vector with
+     dist number of distinct trailing values.  Always possible
+     when dist == 1 or when nunits is constant or when the initializations
+     are uniform.  */
+  tree uniform_initval = NULL_TREE;
+  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
+  for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+    {
+      gphi *phi = as_a <gphi *> (s->stmt);
+      if (! uniform_initval)
+       uniform_initval = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+      else if (! operand_equal_p (uniform_initval,
+                                 PHI_ARG_DEF_FROM_EDGE (phi, pe)))
+       {
+         uniform_initval = NULL_TREE;
+         break;
+       }
+    }
+  if (!uniform_initval && !nunits.is_constant ())
+    {
+      if (dump_enabled_p ())
+       dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+                        "cannot build initialization vector for "
+                        "first order recurrence\n");
+      return false;
+    }
+
   /* First-order recurrence autovectorization needs to handle permutation
      with indices = [nunits-1, nunits, nunits+1, ...].  */
   vec_perm_builder sel (nunits, 1, 3);
@@ -9714,21 +9741,38 @@ vectorizable_recurr (loop_vec_info loop_vinfo, stmt_vec_info stmt_info,
       return true;
     }
 
-  edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
-  basic_block bb = gimple_bb (phi);
-  tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
-  if (!useless_type_conversion_p (TREE_TYPE (vectype), TREE_TYPE (preheader)))
+  tree vec_init;
+  if (! uniform_initval)
     {
-      gimple_seq stmts = NULL;
-      preheader = gimple_convert (&stmts, TREE_TYPE (vectype), preheader);
-      gsi_insert_seq_on_edge_immediate (pe, stmts);
+      vec<constructor_elt, va_gc> *v = NULL;
+      vec_alloc (v, nunits.to_constant ());
+      for (unsigned i = 0; i < nunits.to_constant () - dist; ++i)
+       CONSTRUCTOR_APPEND_ELT (v, NULL_TREE,
+                               build_zero_cst (TREE_TYPE (vectype)));
+      for (stmt_vec_info s : SLP_TREE_SCALAR_STMTS (slp_node))
+       {
+         gphi *phi = as_a <gphi *> (s->stmt);
+         tree preheader = PHI_ARG_DEF_FROM_EDGE (phi, pe);
+         if (!useless_type_conversion_p (TREE_TYPE (vectype),
+                                         TREE_TYPE (preheader)))
+           {
+             gimple_seq stmts = NULL;
+             preheader = gimple_convert (&stmts,
+                                         TREE_TYPE (vectype), preheader);
+             gsi_insert_seq_on_edge_immediate (pe, stmts);
+           }
+         CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, preheader);
+       }
+      vec_init = build_constructor (vectype, v);
     }
-  tree vec_init = build_vector_from_val (vectype, preheader);
+  else
+    vec_init = uniform_initval;
   vec_init = vect_init_vector (loop_vinfo, stmt_info, vec_init, vectype, NULL);
 
   /* Create the vectorized first-order PHI node.  */
   tree vec_dest = vect_get_new_vect_var (vectype,
                                         vect_simple_var, "vec_recur_");
+  basic_block bb = gimple_bb (phi);
   gphi *new_phi = create_phi_node (vec_dest, bb);
   add_phi_arg (new_phi, vec_init, pe, UNKNOWN_LOCATION);