VECT: Support loop len control on EXTRACT_LAST vectorization

Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
powerpc and s390 with your suggestions:

-  machine_mode len_load_mode = get_len_load_store_mode
-    (loop_vinfo->vector_mode, true).require ();
-  machine_mode len_store_mode = get_len_load_store_mode
-    (loop_vinfo->vector_mode, false).require ();
+  machine_mode len_load_mode, len_store_mode;
+  if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
+        .exists (&len_load_mode))
+    return false;
+  if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
+        .exists (&len_store_mode))
+    return false;

Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
gcc/ChangeLog:

	* tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
	(vectorizable_live_operation): Add live vectorization for length loop
	control.

gcc/testsuite/ChangeLog:

	* gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
	* gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
new file mode 100644 (file)
index 0000000..75fa2eb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
+
+#include <stdint-gcc.h>
+
+#define EXTRACT_LAST(TYPE)                                                     \
+  TYPE __attribute__ ((noinline, noclone))                                     \
+  test_##TYPE (TYPE *x, int n, TYPE value)                                     \
+  {                                                                            \
+    TYPE last;                                                                 \
+    for (int j = 0; j < n; ++j)                                                \
+      {                                                                        \
+       last = x[j];                                                           \
+       x[j] = last * value;                                                   \
+      }                                                                        \
+    return last;                                                               \
+  }
+
+#define TEST_ALL(T)                                                            \
+  T (int8_t)                                                                   \
+  T (int16_t)                                                                  \
+  T (int32_t)                                                                  \
+  T (int64_t)                                                                  \
+  T (uint8_t)                                                                  \
+  T (uint16_t)                                                                 \
+  T (uint32_t)                                                                 \
+  T (uint64_t)                                                                 \
+  T (_Float16)                                                                 \
+  T (float)                                                                    \
+  T (double)
+
+TEST_ALL (EXTRACT_LAST)
+
+/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
new file mode 100644 (file)
index 0000000..42913a1
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
@@ -0,0 +1,35 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
+
+#include "live-1.c"
+
+#define N 107
+#define OP 70
+
+#define TEST_LOOP(TYPE)                                \
+  {                                            \
+    TYPE a[N];                                 \
+    for (int i = 0; i < N; ++i)                        \
+      {                                                \
+       a[i] = i * 2 + (i % 3);                 \
+       asm volatile ("" ::: "memory");         \
+      }                                                \
+    TYPE expected = a[N - 1];                  \
+    TYPE res = test_##TYPE (a, N, OP);         \
+    if (res != expected)                       \
+      __builtin_abort ();                      \
+    for (int i = 0; i < N; ++i)                        \
+      {                                                \
+       TYPE old = i * 2 + (i % 3);             \
+       if (a[i] != (TYPE) (old * (TYPE) OP))   \
+         __builtin_abort ();                   \
+       asm volatile ("" ::: "memory");         \
+      }                                                \
+  }
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+  TEST_ALL (TEST_LOOP);
+  return 0;
+}

diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
index 1fcd8d07ea1448920b9fb203d96b5f5cb0c2282a..1cd6c291377729b28f01168e7745a1665073836e 100644 (file)
--- a/gcc/tree-vect-loop.cc
+++ b/gcc/tree-vect-loop.cc
@@ -32,6 +32,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "tree-pass.h"
 #include "ssa.h"
 #include "optabs-tree.h"
+#include "memmodel.h"
+#include "optabs.h"
 #include "diagnostic-core.h"
 #include "fold-const.h"
 #include "stor-layout.h"
@@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
   if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
     return false;
 
-  machine_mode len_load_mode = get_len_load_store_mode
-    (loop_vinfo->vector_mode, true).require ();
-  machine_mode len_store_mode = get_len_load_store_mode
-    (loop_vinfo->vector_mode, false).require ();
+  machine_mode len_load_mode, len_store_mode;
+  if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
+        .exists (&len_load_mode))
+    return false;
+  if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
+        .exists (&len_store_mode))
+    return false;
 
   signed char partial_load_bias = internal_len_load_store_bias
     (IFN_LEN_LOAD, len_load_mode);
@@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
       /* No transformation required.  */
       if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
        {
-         if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
-                                              OPTIMIZE_FOR_SPEED))
-           {
-             if (dump_enabled_p ())
-               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
-                                "can't operate on partial vectors "
-                                "because the target doesn't support extract "
-                                "last reduction.\n");
-             LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
-           }
-         else if (slp_node)
+         if (slp_node)
            {
              if (dump_enabled_p ())
                dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
          else
            {
              gcc_assert (ncopies == 1 && !slp_node);
-             vect_record_loop_mask (loop_vinfo,
-                                    &LOOP_VINFO_MASKS (loop_vinfo),
-                                    1, vectype, NULL);
+             if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
+                                                 OPTIMIZE_FOR_SPEED))
+               vect_record_loop_mask (loop_vinfo,
+                                      &LOOP_VINFO_MASKS (loop_vinfo),
+                                      1, vectype, NULL);
+             else if (can_vec_extract_var_idx_p (
+                        TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
+               vect_record_loop_len (loop_vinfo,
+                                     &LOOP_VINFO_LENS (loop_vinfo),
+                                     1, vectype, 1);
+             else
+               {
+                 if (dump_enabled_p ())
+                   dump_printf_loc (
+                     MSG_MISSED_OPTIMIZATION, vect_location,
+                     "can't operate on partial vectors "
+                     "because the target doesn't support extract "
+                     "last reduction.\n");
+                 LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+               }
            }
        }
       /* ???  Enable for loop costing as well.  */
@@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
   gimple *vec_stmt;
   if (slp_node)
     {
-      gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
+      gcc_assert (!loop_vinfo
+                 || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+                     && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
 
       /* Get the correct slp vectorized stmt.  */
       vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
@@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
 
       gimple_seq stmts = NULL;
       tree new_tree;
-      if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+      if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+       {
+         /* Emit:
+
+              SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
+
+            where VEC_LHS is the vectorized live-out result and MASK is
+            the loop mask for the final iteration.  */
+         gcc_assert (ncopies == 1 && !slp_node);
+         gimple_seq tem = NULL;
+         gimple_stmt_iterator gsi = gsi_last (tem);
+         tree len
+           = vect_get_loop_len (loop_vinfo, &gsi,
+                                &LOOP_VINFO_LENS (loop_vinfo),
+                                1, vectype, 0, 0);
+
+         /* BIAS - 1.  */
+         signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+         tree bias_minus_one
+           = int_const_binop (MINUS_EXPR,
+                              build_int_cst (TREE_TYPE (len), biasval),
+                              build_one_cst (TREE_TYPE (len)));
+
+         /* LAST_INDEX = LEN + (BIAS - 1).  */
+         tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
+                                         len, bias_minus_one);
+
+         /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>.  */
+         tree scalar_res
+           = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
+                           vec_lhs_phi, last_index);
+
+         /* Convert the extracted vector element to the scalar type.  */
+         new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
+       }
+      else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
        {
          /* Emit: