[AutoFDO] Add hierarchical discriminator for vectorizer

Add hierarchical discriminator support for vectorized loop versioning.
Assigns copyid discriminators to distinguish vectorized and scalar versions.

gcc/ChangeLog:

	* tree-vect-loop-manip.cc (vect_loop_versioning): Assign copyid
	discriminators for vectorized and scalar loop versions.
	* tree-vect-loop.cc (vect_transform_loop): Assign copyid
	discriminators for vectorized and scalar loop versions.

gcc/testsuite/ChangeLog:

	* gcc.dg/hierarchical-discriminator-vect-version.c: New test.

Signed-off-by: Kugan Vivekanandarajah <kvivekananda@nvidia.com>

diff --git a/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c b/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c
new file mode 100644 (file)
index 0000000..5dc945f
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/hierarchical-discriminator-vect-version.c
@@ -0,0 +1,17 @@
+/* Test that vectorization assigns multiplicity and copyid discriminators.
+   { dg-do compile }
+   { dg-options "-O3 -g -ftree-vectorize -fno-vect-cost-model" }
+   { dg-require-effective-target vect_int }
+    */
+
+void
+test_vectorize (int *a, int *b, int *c, int n)
+{
+  int i;
+  for (i = 0; i < n; i++)
+    {
+      a[i] = b[i] + c[i];  /* Main vectorized loop and epilog */
+    }
+}
+
+/* { dg-final { scan-assembler "\\.loc 1 13 \[0-9\]+ is_stmt 0 discriminator \[0-9\]+" } } */

diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
index 0719e2cf493d3629c9522653f3d21b24e16282dc..f28d89ce2d10c74aaf9ce556ac1b94adf7ea9d56 100644 (file)
--- a/gcc/tree-vect-loop-manip.cc
+++ b/gcc/tree-vect-loop-manip.cc
@@ -53,6 +53,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "langhooks.h"
 #include "tree-vector-builder.h"
 #include "optabs-tree.h"
+#include "hierarchical_discriminator.h"
+
 
 /*************************************************************************
   Simple Loop Peeling Utilities
@@ -3464,6 +3466,15 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
       gcc_assert (prolog);
       prolog->force_vectorize = false;
 
+      /* Assign hierarchical discriminators to distinguish prolog loop.  */
+      gimple *prolog_last = last_nondebug_stmt (prolog->header);
+      location_t prolog_loc
+               = prolog_last ? gimple_location (prolog_last) : UNKNOWN_LOCATION;
+      if (prolog_loc != UNKNOWN_LOCATION)
+       {
+         unsigned int prolog_copyid = allocate_copyid_base (prolog_loc, 1);
+         assign_discriminators_to_loop (prolog, 0, prolog_copyid);
+       }
       first_loop = prolog;
       reset_original_copy_tables ();
 
@@ -3573,6 +3584,22 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
       epilog->force_vectorize = false;
       bb_before_epilog = loop_preheader_edge (epilog)->src;
 
+      /* Assign hierarchical discriminators to distinguish epilog loop.
+        Only assign if it's a scalar epilog.  If it will be vectorized
+        (vect_epilogues), discriminators will be assigned.
+        Use dynamic copy_id allocation instead of hardcoded constants.  */
+      if (!vect_epilogues)
+       {
+         gimple *epilog_last = last_nondebug_stmt (epilog->header);
+         location_t epilog_loc
+           = epilog_last ? gimple_location (epilog_last) : UNKNOWN_LOCATION;
+         if (epilog_loc != UNKNOWN_LOCATION)
+           {
+             unsigned int epilog_copyid = allocate_copyid_base (epilog_loc, 1);
+             assign_discriminators_to_loop (epilog, 0, epilog_copyid);
+           }
+       }
+
       /* Scalar version loop may be preferred.  In this case, add guard
         and skip to epilog.  Note this only happens when the number of
         iterations of loop is unknown at compile time, otherwise this
@@ -4501,6 +4528,18 @@ vect_loop_versioning (loop_vec_info loop_vinfo,
       gcc_assert (nloop);
       nloop = get_loop_copy (loop);
 
+      /* Assign hierarchical discriminators to distinguish loop versions.
+        Only assign to the scalar version here; the vectorized version will
+        get discriminators later during transformation/peeling.
+        Use dynamic copy_id allocation instead of hardcoded constants.  */
+      gimple *nloop_last = last_nondebug_stmt (nloop->header);
+      location_t nloop_loc
+               = nloop_last ? gimple_location (nloop_last) : UNKNOWN_LOCATION;
+      if (nloop_loc != UNKNOWN_LOCATION)
+       {
+         unsigned int nloop_copyid = allocate_copyid_base (nloop_loc, 1);
+         assign_discriminators_to_loop (nloop, 0, nloop_copyid);
+       }
       /* For cycle vectorization with SLP we rely on the PHI arguments
         appearing in the same order as the SLP node operands which for the
         loop PHI nodes means the preheader edge dest index needs to remain

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index fdf544fa47b043afe68e0d5f8a65f04014b7836f..9c2345ce955d12cc7d074f564a9f28b98ef626da 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -59,6 +59,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "case-cfn-macros.h"
 #include "langhooks.h"
 #include "opts.h"
+#include "hierarchical_discriminator.h"
 
 /* Loop Vectorization Pass.
 
@@ -11210,6 +11211,23 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
                              &step_vector, &niters_vector_mult_vf, th,
                              check_profitability, niters_no_overflow,
                              &advance);
+
+  /* Assign hierarchical discriminators to the vectorized loop.  */
+  poly_uint64 vf_val = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+  unsigned int vf_int = constant_lower_bound (vf_val);
+  if (vf_int > DISCR_MULTIPLICITY_MAX)
+    vf_int = DISCR_MULTIPLICITY_MAX;
+
+  /* Assign unique copy_id dynamically instead of using hardcoded constants.
+     Epilogue and main vectorized loops get different copy_ids.  */
+  gimple *loop_last = last_nondebug_stmt (loop->header);
+  location_t loop_loc
+    = loop_last ? gimple_location (loop_last) : UNKNOWN_LOCATION;
+  if (loop_loc != UNKNOWN_LOCATION)
+    {
+      unsigned int copyid = allocate_copyid_base (loop_loc, 1);
+      assign_discriminators_to_loop (loop, vf_int, copyid);
+    }
   if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
       && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
     {