AArch64: Add pattern for unsigned widenings (uxtl) to zip{1,2}

This changes unpack instructions to use zip{1,2} when doing a zero-extending
widening operation.  Permutes generally have a higher throughput than the
widening operations. Zeros are shuffled into the top half of the registers.

The testcase

void d2 (unsigned * restrict a, unsigned short *b, int n)
{
    for (int i = 0; i < (n & -8); i++)
      a[i] = b[i];
}

now generates:

        movi    v1.4s, 0
.L3:
        ldr     q0, [x1], 16
        zip1    v2.8h, v0.8h, v1.8h
        zip2    v0.8h, v0.8h, v1.8h
        stp     q2, q0, [x0]
        add     x0, x0, 32
        cmp     x1, x2
        bne     .L3

instead of:

.L3:
        ldr     q0, [x1], 16
        uxtl    v1.4s, v0.4h
        uxtl2   v0.4s, v0.8h
        stp     q1, q0, [x0]
        add     x0, x0, 32
        cmp     x1, x2
        bne     .L3

Since we need the extra 0 register we do this only for the vectorizer's lo/hi
pairs when we know the 0 will be floated outside of the loop.

This gives an 8% speed-up in Imagick in SPECCPU 2017 on Neoverse V2.

gcc/ChangeLog:

	* config/aarch64/aarch64-simd.md (vec_unpack<su>_lo_<mode,
	vec_unpack<su>_lo_<mode): Split into...
	(vec_unpacku_lo_<mode, vec_unpacks_lo_<mode,
	vec_unpacku_lo_<mode, vec_unpacks_lo_<mode): ...These.
	(aarch64_usubw<mode>_<PERM_EXTEND:perm_hilo>_zip): New.
	(aarch64_uaddw<mode>_<PERM_EXTEND:perm_hilo>_zip): New.
	* config/aarch64/iterators.md (PERM_EXTEND, perm_index): New.
	(perm_hilo): Add UNSPEC_ZIP1, UNSPEC_ZIP2.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/simd/vmovl_high_1.c: Update codegen.
	* gcc.target/aarch64/uxtl-combine-1.c: New test.
	* gcc.target/aarch64/uxtl-combine-2.c: New test.
	* gcc.target/aarch64/uxtl-combine-3.c: New test.
	* gcc.target/aarch64/uxtl-combine-4.c: New test.
	* gcc.target/aarch64/uxtl-combine-5.c: New test.
	* gcc.target/aarch64/uxtl-combine-6.c: New test.

diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index c6f2d5828373f2a5272b9d1227bfe34365f9fd09..75ee659871080ed28b9887990b7431682c283502 100644 (file)
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -2029,26 +2029,60 @@
   [(set_attr "type" "neon_shift_imm_long")]
 )
 
-(define_expand "vec_unpack<su>_hi_<mode>"
+(define_expand "vec_unpacku_hi_<mode>"
   [(match_operand:<VWIDE> 0 "register_operand")
-   (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
+   (match_operand:VQW 1 "register_operand")]
+  "TARGET_SIMD"
+  {
+    rtx res = gen_reg_rtx (<MODE>mode);
+    rtx tmp = aarch64_gen_shareable_zero (<MODE>mode);
+    if (BYTES_BIG_ENDIAN)
+      emit_insn (gen_aarch64_zip2<mode> (res, tmp, operands[1]));
+    else
+     emit_insn (gen_aarch64_zip2<mode> (res, operands[1], tmp));
+    emit_move_insn (operands[0],
+                  simplify_gen_subreg (<VWIDE>mode, res, <MODE>mode, 0));
+    DONE;
+  }
+)
+
+(define_expand "vec_unpacks_hi_<mode>"
+  [(match_operand:<VWIDE> 0 "register_operand")
+   (match_operand:VQW 1 "register_operand")]
   "TARGET_SIMD"
   {
     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
-    emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
-                                                         operands[1], p));
+    emit_insn (gen_aarch64_simd_vec_unpacks_hi_<mode> (operands[0],
+                                                      operands[1], p));
+    DONE;
+  }
+)
+
+(define_expand "vec_unpacku_lo_<mode>"
+  [(match_operand:<VWIDE> 0 "register_operand")
+   (match_operand:VQW 1 "register_operand")]
+  "TARGET_SIMD"
+  {
+    rtx res = gen_reg_rtx (<MODE>mode);
+    rtx tmp = aarch64_gen_shareable_zero (<MODE>mode);
+    if (BYTES_BIG_ENDIAN)
+       emit_insn (gen_aarch64_zip1<mode> (res, tmp, operands[1]));
+    else
+       emit_insn (gen_aarch64_zip1<mode> (res, operands[1], tmp));
+    emit_move_insn (operands[0],
+                  simplify_gen_subreg (<VWIDE>mode, res, <MODE>mode, 0));
     DONE;
   }
 )
 
-(define_expand "vec_unpack<su>_lo_<mode>"
+(define_expand "vec_unpacks_lo_<mode>"
   [(match_operand:<VWIDE> 0 "register_operand")
-   (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
+   (match_operand:VQW 1 "register_operand")]
   "TARGET_SIMD"
   {
     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
-    emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
-                                                         operands[1], p));
+    emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
+                                                      operands[1], p));
     DONE;
   }
 )
@@ -4776,6 +4810,34 @@
   [(set_attr "type" "neon_sub_widen")]
 )
 
+(define_insn "aarch64_usubw<mode>_<PERM_EXTEND:perm_hilo>_zip"
+  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+       (minus:<VWIDE>
+         (match_operand:<VWIDE> 1 "register_operand" "w")
+         (subreg:<VWIDE>
+           (unspec:<MODE> [
+               (match_operand:VQW 2 "register_operand" "w")
+               (match_operand:VQW 3 "aarch64_simd_imm_zero")
+              ] PERM_EXTEND) 0)))]
+  "TARGET_SIMD"
+  "usubw<PERM_EXTEND:perm_index>\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
+  [(set_attr "type" "neon_sub_widen")]
+)
+
+(define_insn "aarch64_uaddw<mode>_<PERM_EXTEND:perm_hilo>_zip"
+  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
+       (plus:<VWIDE>
+         (subreg:<VWIDE>
+           (unspec:<MODE> [
+               (match_operand:VQW 2 "register_operand" "w")
+               (match_operand:VQW 3 "aarch64_simd_imm_zero")
+              ] PERM_EXTEND) 0)
+         (match_operand:<VWIDE> 1 "register_operand" "w")))]
+  "TARGET_SIMD"
+  "uaddw<PERM_EXTEND:perm_index>\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
+  [(set_attr "type" "neon_add_widen")]
+)
+
 (define_insn "aarch64_<ANY_EXTEND:su>addw<mode>"
   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
        (plus:<VWIDE>

diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index a920de99ffca378ce518f378a35cbe2766877ee8..2354315d7d249ccee46625d13b32678f1da1f087 100644 (file)
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -2645,6 +2645,9 @@
 (define_int_iterator OPTAB_PERMUTE [UNSPEC_ZIP1 UNSPEC_ZIP2
                                    UNSPEC_UZP1 UNSPEC_UZP2])
 
+;; Permutes for zero extends
+(define_int_iterator PERM_EXTEND [UNSPEC_ZIP1 UNSPEC_ZIP2])
+
 (define_int_iterator REVERSE [UNSPEC_REV64 UNSPEC_REV32 UNSPEC_REV16])
 
 (define_int_iterator FRINT [UNSPEC_FRINTZ UNSPEC_FRINTP UNSPEC_FRINTM
@@ -3467,7 +3470,10 @@
                         (UNSPEC_REV16 "16")])
 
 (define_int_attr perm_hilo [(UNSPEC_UNPACKSHI "hi") (UNSPEC_UNPACKUHI "hi")
-                           (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo")])
+                           (UNSPEC_UNPACKSLO "lo") (UNSPEC_UNPACKULO "lo")
+                           (UNSPEC_ZIP2 "hi") (UNSPEC_ZIP1 "lo")])
+
+(define_int_attr perm_index [(UNSPEC_ZIP2 "2") (UNSPEC_ZIP1 "")])
 
 ;; Return true if the associated optab refers to the high-numbered lanes,
 ;; false if it refers to the low-numbered lanes.  The convention is for

diff --git a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
index d45bb83e3503d512c443f37a446d30d188719a96..a2d09eaee0de5a3d3409330c5c26a3b5315e84eb 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
+++ b/gcc/testsuite/gcc.target/aarch64/simd/vmovl_high_1.c
@@ -22,11 +22,11 @@ FUNC (int32x4_t, int64x2_t, s32)
 /* { dg-final { scan-assembler-times {sxtl2\tv0\.2d, v0\.4s} 1} }  */
 
 FUNC (uint8x16_t, uint16x8_t, u8)
-/* { dg-final { scan-assembler-times {uxtl2\tv0\.8h, v0\.16b} 1} }  */
+/* { dg-final { scan-assembler-times {zip2\tv0\.16b, v0\.16b} 1} }  */
 
 FUNC (uint16x8_t, uint32x4_t, u16)
-/* { dg-final { scan-assembler-times {uxtl2\tv0\.4s, v0\.8h} 1} }  */
+/* { dg-final { scan-assembler-times {zip2\tv0\.8h, v0\.8h} 1} }  */
 
 FUNC (uint32x4_t, uint64x2_t, u32)
-/* { dg-final { scan-assembler-times {uxtl2\tv0\.2d, v0\.4s} 1} }  */
+/* { dg-final { scan-assembler-times {zip2\tv0\.4s, v0\.4s} 1} }  */
 

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c
new file mode 100755 (executable)
index 0000000..68fa9a0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-1.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN unsigned
+#define TYPE1 char
+#define TYPE2 short
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
+/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
+/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c
new file mode 100755 (executable)
index 0000000..af8a890
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-2.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN unsigned
+#define TYPE1 short
+#define TYPE2 int
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
+/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
+/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c
new file mode 100755 (executable)
index 0000000..cdae6d0
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-3.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN unsigned
+#define TYPE1 int
+#define TYPE2 long long
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-times {\tzip1\t} 1 } } */
+/* { dg-final { scan-assembler-times {\tzip2\t} 1 } } */
+/* { dg-final { scan-assembler-not {\tuxtl\t} } } */
+/* { dg-final { scan-assembler-not {\tuxtl2\t} } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c
new file mode 100755 (executable)
index 0000000..e1a9c4f
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-4.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN signed
+#define TYPE1 char
+#define TYPE2 short
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-not {\tzip1\t} } } */
+/* { dg-final { scan-assembler-not {\tzip2\t} } } */
+/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
+/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c
new file mode 100755 (executable)
index 0000000..92b09ba
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-5.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN signed
+#define TYPE1 short
+#define TYPE2 int
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-not {\tzip1\t} } } */
+/* { dg-final { scan-assembler-not {\tzip2\t} } } */
+/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
+/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c
new file mode 100755 (executable)
index 0000000..5c6e635
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/uxtl-combine-6.c
@@ -0,0 +1,20 @@
+/* { dg-do assemble } */
+/* { dg-options "-O3 --save-temps --param=vect-epilogues-nomask=0" } */
+
+#pragma GCC target "+nosve"
+
+#define SIGN signed
+#define TYPE1 int
+#define TYPE2 long long
+
+void d2 (SIGN TYPE2 * restrict a, SIGN TYPE1 *b, int n)
+{
+    for (int i = 0; i < (n & -8); i++)
+      a[i] = b[i];
+}
+
+/* { dg-final { scan-assembler-not {\tzip1\t} } } */
+/* { dg-final { scan-assembler-not {\tzip2\t} } } */
+/* { dg-final { scan-assembler-times {\tsxtl\t} 1 } } */
+/* { dg-final { scan-assembler-time {\tsxtl2\t} 1 } } */
+