RISC-V: Support interleave vector with different step sequence

This patch fixes 64 ICEs in full coverage testing since they happens due to same reason.

Before this patch:

internal compiler error: in expand_const_vector, at config/riscv/riscv-v.cc:1270

appears 400 times in full coverage testing report.

The root cause is we didn't support interleave vector with different steps.

Here is the story:

We already supported interleave with single same step, that is:
e.g. v = { 0, 100, 2, 102, 4, 104, ... }
This sequence can be interpreted as interleave vector by 2 seperate sequences:
sequence1 = { 0, 2, 4, ... } and sequence2 = { 100, 102, 104, ... }.
Their step are both 2.

However, we didn't support interleave vector when they have different steps which
cause ICE in such situations.

This patch support different steps interleaved vector for the following 2 situations:

1. When vector can be extended EEW:

Case 1: { 0, 0, 1, 0, 2, 0, ... }
It's interleaved by sequence1 = { 0, 1, 2, ... } and sequence1 = { 0, 0, 0, ... }
Suppose the original vector can be extended EEW, e.g. mode = RVVM1SImode.
Then such interleaved vector can be achieved with { 1, 2, 3, ... } with RVVM1DImode.
So, for this situation the codegen is pretty efficient and clean:

.MASK_LEN_STORE (&s, 32B, { -1, ... }, 16, 0, { 0, 0, 1, 0, 2, 0, ... });

->
   vsetvli	a5,zero,e64,m8,ta,ma
   vid.v	v8
   vsetivli	zero,16,e32,m8,ta,ma
   vse32.v	v8,0(a4)

Case 2: { 0, 100, 1, 100, 2, 100, ... }

.MASK_LEN_STORE (&s, 32B, { -1, ... }, 16, 0, { 0, 100, 1, 100, 2, 100, ... });

->

   vsetvli	a1,zero,e64,m8,ta,ma
   vid.v	v8
   li	a7,100
   vand.vx	v8,v8,a4
   vsetivli	zero,16,e32,m8,ta,ma
   vse32.v	v8,0(a5)

2. When vector can't be extended EEW:

Since we can't use EEW = 64, for example, RVVM1SImode in -march=rv32gc_zve32f,
we use vmerge to combine the sequence.

.MASK_LEN_STORE (&s, 32B, { -1, ... }, 16, 0, { 200, 100, 201, 103, 202, 106, ... });

1. Generate sequence1 = { 200, 200, 201, 201, 202, 202, ... } and sequence2 = { 100, 100, 103, 103, 106, 106, ... }
2. Merge sequence1 and sequence2 with mask { 0, 1, 0, 1, ... }

gcc/ChangeLog:

	* config/riscv/riscv-protos.h (expand_vec_series): Adapt function.
	* config/riscv/riscv-v.cc (rvv_builder::double_steps_npatterns_p): New function.
	(expand_vec_series): Adapt function.
	(expand_const_vector): Support new interleave vector with different step.

gcc/testsuite/ChangeLog:

	* gcc.target/riscv/rvv/autovec/slp-interleave-1.c: New test.
	* gcc.target/riscv/rvv/autovec/slp-interleave-2.c: New test.
	* gcc.target/riscv/rvv/autovec/slp-interleave-3.c: New test.
	* gcc.target/riscv/rvv/autovec/slp-interleave-4.c: New test.

diff --git a/gcc/config/riscv/riscv-protos.h b/gcc/config/riscv/riscv-protos.h
index bfbd2bf0d18b14169a063e1ce0d7e54fd938679f..a6f204f3066ffcbf9406948be3e342d56fc10152 100644 (file)
--- a/gcc/config/riscv/riscv-protos.h
+++ b/gcc/config/riscv/riscv-protos.h
@@ -543,7 +543,7 @@ void expand_tuple_move (rtx *);
 bool expand_block_move (rtx, rtx, rtx);
 machine_mode preferred_simd_mode (scalar_mode);
 machine_mode get_mask_mode (machine_mode);
-void expand_vec_series (rtx, rtx, rtx);
+void expand_vec_series (rtx, rtx, rtx, rtx = 0);
 void expand_vec_init (rtx, rtx);
 void expand_vec_perm (rtx, rtx, rtx, rtx);
 void expand_select_vl (rtx *);

diff --git a/gcc/config/riscv/riscv-v.cc b/gcc/config/riscv/riscv-v.cc
index 71cb7567f1a5933cf86a2c6dcd019215de73512e..9b99d0aca84424db07f9948f5ef542a3d1ee451a 100644 (file)
--- a/gcc/config/riscv/riscv-v.cc
+++ b/gcc/config/riscv/riscv-v.cc
@@ -432,6 +432,7 @@ public:
 
   bool single_step_npatterns_p () const;
   bool npatterns_all_equal_p () const;
+  bool interleaved_stepped_npatterns_p () const;
 
   machine_mode new_mode () const { return m_new_mode; }
   scalar_mode inner_mode () const { return m_inner_mode; }
@@ -668,6 +669,27 @@ rvv_builder::single_step_npatterns_p () const
   return true;
 }
 
+/* Return true if the permutation consists of two
+   interleaved patterns with a constant step each.
+   TODO: We currently only support NPATTERNS = 2.  */
+bool
+rvv_builder::interleaved_stepped_npatterns_p () const
+{
+  if (npatterns () != 2 || nelts_per_pattern () != 3)
+    return false;
+  for (unsigned int i = 0; i < npatterns (); i++)
+    {
+      poly_int64 ele0 = rtx_to_poly_int64 (elt (i));
+      poly_int64 ele1 = rtx_to_poly_int64 (elt (npatterns () + i));
+      poly_int64 ele2 = rtx_to_poly_int64 (elt (npatterns () * 2 + i));
+      poly_int64 diff1 = ele1 - ele0;
+      poly_int64 diff2 = ele2 - ele1;
+      if (maybe_ne (diff1, diff2))
+       return false;
+    }
+  return true;
+}
+
 /* Return true if all elements of NPATTERNS are equal.
 
    E.g. NPATTERNS = 4:
@@ -955,10 +977,15 @@ get_repeating_sequence_dup_machine_mode (const rvv_builder &builder,
   return get_vector_mode (inner_mode, dup_nunit).require ();
 }
 
-/* Expand series const vector.  */
+/* Expand series const vector.  If VID is NULL_RTX, we use vid.v
+   instructions to generate sequence for VID:
+
+     VID = { 0, 1, 2, 3, ... }
+
+   Otherwise, we use the VID argument directly.  */
 
 void
-expand_vec_series (rtx dest, rtx base, rtx step)
+expand_vec_series (rtx dest, rtx base, rtx step, rtx vid)
 {
   machine_mode mode = GET_MODE (dest);
   poly_int64 nunits_m1 = GET_MODE_NUNITS (mode) - 1;
@@ -968,14 +995,18 @@ expand_vec_series (rtx dest, rtx base, rtx step)
   /* VECT_IV = BASE + I * STEP.  */
 
   /* Step 1: Generate I = { 0, 1, 2, ... } by vid.v.  */
-  rtx vid = gen_reg_rtx (mode);
-  rtx op[] = {vid};
-  emit_vlmax_insn (code_for_pred_series (mode), NULLARY_OP, op);
+  bool reverse_p = !vid && rtx_equal_p (step, constm1_rtx)
+                  && poly_int_rtx_p (base, &value)
+                  && known_eq (nunits_m1, value);
+  if (!vid)
+    {
+      vid = gen_reg_rtx (mode);
+      rtx op[] = {vid};
+      emit_vlmax_insn (code_for_pred_series (mode), NULLARY_OP, op);
+    }
 
   rtx step_adj;
-  if (rtx_equal_p (step, constm1_rtx)
-      && poly_int_rtx_p (base, &value)
-      && known_eq (nunits_m1, value))
+  if (reverse_p)
     {
       /* Special case:
           {nunits - 1, nunits - 2, ... , 0}.
@@ -1246,13 +1277,108 @@ expand_const_vector (rtx target, rtx src)
                                BINARY_OP, add_ops);
            }
        }
+      else if (builder.interleaved_stepped_npatterns_p ())
+       {
+         rtx base1 = builder.elt (0);
+         rtx base2 = builder.elt (1);
+         poly_int64 step1
+           = rtx_to_poly_int64 (builder.elt (builder.npatterns ()))
+             - rtx_to_poly_int64 (base1);
+         poly_int64 step2
+           = rtx_to_poly_int64 (builder.elt (builder.npatterns () + 1))
+             - rtx_to_poly_int64 (base2);
+
+         /* For { 1, 0, 2, 0, ... , n - 1, 0 }, we can use larger EEW
+            integer vector mode to generate such vector efficiently.
+
+            E.g. EEW = 16, { 2, 0, 4, 0, ... }
+
+            can be interpreted into:
+
+                 EEW = 32, { 2, 4, ... }  */
+         unsigned int new_smode_bitsize = builder.inner_bits_size () * 2;
+         scalar_int_mode new_smode;
+         machine_mode new_mode;
+         poly_uint64 new_nunits
+           = exact_div (GET_MODE_NUNITS (builder.mode ()), 2);
+         if (int_mode_for_size (new_smode_bitsize, 0).exists (&new_smode)
+             && get_vector_mode (new_smode, new_nunits).exists (&new_mode))
+           {
+             rtx tmp = gen_reg_rtx (new_mode);
+             base1 = gen_int_mode (rtx_to_poly_int64 (base1), new_smode);
+             expand_vec_series (tmp, base1, gen_int_mode (step1, new_smode));
+
+             if (rtx_equal_p (base2, const0_rtx) && known_eq (step2, 0))
+               /* { 1, 0, 2, 0, ... }.  */
+               emit_move_insn (target, gen_lowpart (mode, tmp));
+             else if (known_eq (step2, 0))
+               {
+                 /* { 1, 1, 2, 1, ... }.  */
+                 rtx scalar = expand_simple_binop (
+                   new_smode, ASHIFT,
+                   gen_int_mode (rtx_to_poly_int64 (base2), new_smode),
+                   gen_int_mode (builder.inner_bits_size (), new_smode),
+                   NULL_RTX, false, OPTAB_DIRECT);
+                 rtx tmp2 = gen_reg_rtx (new_mode);
+                 rtx and_ops[] = {tmp2, tmp, scalar};
+                 emit_vlmax_insn (code_for_pred_scalar (AND, new_mode),
+                                  BINARY_OP, and_ops);
+                 emit_move_insn (target, gen_lowpart (mode, tmp2));
+               }
+             else
+               {
+                 /* { 1, 3, 2, 6, ... }.  */
+                 rtx tmp2 = gen_reg_rtx (new_mode);
+                 base2 = gen_int_mode (rtx_to_poly_int64 (base2), new_smode);
+                 expand_vec_series (tmp2, base2,
+                                    gen_int_mode (step1, new_smode));
+                 rtx shifted_tmp2 = expand_simple_binop (
+                   new_mode, ASHIFT, tmp2,
+                   gen_int_mode (builder.inner_bits_size (), Pmode), NULL_RTX,
+                   false, OPTAB_DIRECT);
+                 rtx tmp3 = gen_reg_rtx (new_mode);
+                 rtx ior_ops[] = {tmp3, tmp, shifted_tmp2};
+                 emit_vlmax_insn (code_for_pred (IOR, new_mode), BINARY_OP,
+                                  ior_ops);
+                 emit_move_insn (target, gen_lowpart (mode, tmp3));
+               }
+           }
+         else
+           {
+             rtx vid = gen_reg_rtx (mode);
+             expand_vec_series (vid, const0_rtx, const1_rtx);
+             /* Transform into { 0, 0, 1, 1, 2, 2, ... }.  */
+             rtx shifted_vid
+               = expand_simple_binop (mode, LSHIFTRT, vid, const1_rtx,
+                                      NULL_RTX, false, OPTAB_DIRECT);
+             rtx tmp1 = gen_reg_rtx (mode);
+             rtx tmp2 = gen_reg_rtx (mode);
+             expand_vec_series (tmp1, base1,
+                                gen_int_mode (step1, builder.inner_mode ()),
+                                shifted_vid);
+             expand_vec_series (tmp2, base2,
+                                gen_int_mode (step2, builder.inner_mode ()),
+                                shifted_vid);
+
+             /* Transform into { 0, 1, 0, 1, 0, 1, ... }.  */
+             rtx and_vid = gen_reg_rtx (mode);
+             rtx and_ops[] = {and_vid, vid, const1_rtx};
+             emit_vlmax_insn (code_for_pred_scalar (AND, mode), BINARY_OP,
+                              and_ops);
+             rtx mask = gen_reg_rtx (builder.mask_mode ());
+             expand_vec_cmp (mask, EQ, and_vid, CONST1_RTX (mode));
+
+             rtx ops[] = {target, tmp1, tmp2, mask};
+             emit_vlmax_insn (code_for_pred_merge (mode), MERGE_OP, ops);
+           }
+       }
       else if (npatterns == 1 && nelts_per_pattern == 3)
        {
          /* Generate the following CONST_VECTOR:
             { base0, base1, base1 + step, base1 + step * 2, ... }  */
-         rtx base0 = CONST_VECTOR_ELT (src, 0);
-         rtx base1 = CONST_VECTOR_ELT (src, 1);
-         rtx step = CONST_VECTOR_ELT (src, 2);
+         rtx base0 = builder.elt (0);
+         rtx base1 = builder.elt (1);
+         rtx step = builder.elt (2);
          /* Step 1 - { base1, base1 + step, base1 + step * 2, ... }  */
          rtx tmp = gen_reg_rtx (mode);
          expand_vec_series (tmp, base1, step);

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-1.c
new file mode 100644 (file)
index 0000000..9f37143
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-1.c
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv_zvl1024b -mabi=lp64d -fno-vect-cost-model --param=riscv-autovec-lmul=m8 -O3 -fdump-tree-optimized-details" } */
+
+struct S { int a, b; } s[8];
+
+void
+foo ()
+{
+  int i;
+  for (i = 0; i < 8; i++)
+    {
+      s[i].b = 0;
+      s[i].a = i;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 0, 0, 1, 0, 2, 0, ... \}" 1 "optimized" } } */

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-2.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-2.c
new file mode 100644 (file)
index 0000000..6cc390c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-2.c
@@ -0,0 +1,18 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv_zvl1024b -mabi=lp64d -fno-vect-cost-model --param=riscv-autovec-lmul=m8 -O3 -fdump-tree-optimized-details" } */
+
+struct S { int a, b; } s[8];
+
+void
+foo ()
+{
+  int i;
+  for (i = 0; i < 8; i++)
+    {
+      s[i].b = 1;
+      s[i].a = i;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 0, 1, 1, 1, 2, 1, ... \}" 1 "optimized" } } */
+/* { dg-final { scan-assembler-times {slli\t[a-x0-9]+,\s*[a-x0-9]+,\s*32} 1 } } */

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-3.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-3.c
new file mode 100644 (file)
index 0000000..326d66e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-3.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gcv_zvl1024b -mabi=lp64d -fno-vect-cost-model --param=riscv-autovec-lmul=m8 -O3 -fdump-tree-optimized-details" } */
+
+struct S { int a, b; } s[8];
+
+void
+foo ()
+{
+  int i;
+  for (i = 0; i < 8; i++)
+    {
+      s[i].b = i*3 + 100;
+      s[i].a = i + 200;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 200, 100, 201, 103, 202, 106, ... \}" 1 "optimized" } } */
+/* { dg-final { scan-assembler-times {vsll\.vx} 1 } } */
+/* { dg-final { scan-assembler-times {vor\.vv} 1 } } */

diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-4.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-4.c
new file mode 100644 (file)
index 0000000..2bb73eb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/slp-interleave-4.c
@@ -0,0 +1,19 @@
+/* { dg-do compile } */
+/* { dg-options "-march=rv64gc_zve32f_zvl1024b -mabi=lp64d -fno-vect-cost-model --param=riscv-autovec-lmul=m8 -O3 -fdump-tree-optimized-details" } */
+
+struct S { int a, b; } s[8];
+
+void
+foo ()
+{
+  int i;
+  for (i = 0; i < 8; i++)
+    {
+      s[i].b = i*3 + 100;
+      s[i].a = i + 200;
+    }
+}
+
+/* { dg-final { scan-tree-dump-times "\{ 200, 100, 201, 103, 202, 106, ... \}" 1 "optimized" } } */
+/* { dg-final { scan-assembler-times {vand\.vi} 1 } } */
+/* { dg-final { scan-assembler-times {vmseq\.vi} 1 } } */