AArch64: Fold LD1/ST1 with ptrue to LDR/STR for 128-bit VLS

If -msve-vector-bits=128, SVE loads and stores (LD1 and ST1) with a
ptrue predicate can be replaced by neon instructions (LDR and STR),
thus avoiding the predicate altogether. This also enables formation of
LDP/STP pairs.

For example, the test cases

svfloat64_t
ptrue_load (float64_t *x)
{
  svbool_t pg = svptrue_b64 ();
  return svld1_f64 (pg, x);
}
void
ptrue_store (float64_t *x, svfloat64_t data)
{
  svbool_t pg = svptrue_b64 ();
  return svst1_f64 (pg, x, data);
}

were previously compiled to
(with -O2 -march=armv8.2-a+sve -msve-vector-bits=128):

ptrue_load:
        ptrue   p3.b, vl16
        ld1d    z0.d, p3/z, [x0]
        ret
ptrue_store:
        ptrue   p3.b, vl16
        st1d    z0.d, p3, [x0]
        ret

Now the are compiled to:

ptrue_load:
        ldr     q0, [x0]
        ret
ptrue_store:
        str     q0, [x0]
        ret

The implementation includes the if-statement
if (known_eq (GET_MODE_SIZE (mode), 16)
    && aarch64_classify_vector_mode (mode) == VEC_SVE_DATA)
which checks for 128-bit VLS and excludes partial modes with a
mode size < 128 (e.g. VNx2QI).

The patch was bootstrapped and tested on aarch64-linux-gnu, no regression.
OK for mainline?

Signed-off-by: Jennifer Schmitz <jschmitz@nvidia.com>
gcc/
	* config/aarch64/aarch64.cc (aarch64_emit_sve_pred_move):
	Fold LD1/ST1 with ptrue to LDR/STR for 128-bit VLS.

gcc/testsuite/
	* gcc.target/aarch64/sve/ldst_ptrue_128_to_neon.c: New test.
	* gcc.target/aarch64/sve/cond_arith_6.c: Adjust expected outcome.
	* gcc.target/aarch64/sve/pcs/return_4_128.c: Likewise.
	* gcc.target/aarch64/sve/pcs/return_5_128.c: Likewise.
	* gcc.target/aarch64/sve/pcs/struct_3_128.c: Likewise.

diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index f7bccf532f89fb041909cac3886dd0776aa52159..fff8d9da49d36e2a9cb69ea259ad18bb7b8e83ee 100644 (file)
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -6416,13 +6416,30 @@ aarch64_stack_protect_canary_mem (machine_mode mode, rtx decl_rtl,
 void
 aarch64_emit_sve_pred_move (rtx dest, rtx pred, rtx src)
 {
-  expand_operand ops[3];
   machine_mode mode = GET_MODE (dest);
-  create_output_operand (&ops[0], dest, mode);
-  create_input_operand (&ops[1], pred, GET_MODE(pred));
-  create_input_operand (&ops[2], src, mode);
-  temporary_volatile_ok v (true);
-  expand_insn (code_for_aarch64_pred_mov (mode), 3, ops);
+  if ((MEM_P (dest) || MEM_P (src))
+      && known_eq (GET_MODE_SIZE (mode), 16)
+      && aarch64_classify_vector_mode (mode) == VEC_SVE_DATA
+      && !BYTES_BIG_ENDIAN)
+    {
+      if (MEM_P (src))
+       {
+         rtx tmp = force_reg (V16QImode, adjust_address (src, V16QImode, 0));
+         emit_move_insn (dest, lowpart_subreg (mode, tmp, V16QImode));
+       }
+      else
+       emit_move_insn (adjust_address (dest, V16QImode, 0),
+                       force_lowpart_subreg (V16QImode, src, mode));
+    }
+  else
+    {
+      expand_operand ops[3];
+      create_output_operand (&ops[0], dest, mode);
+      create_input_operand (&ops[1], pred, GET_MODE(pred));
+      create_input_operand (&ops[2], src, mode);
+      temporary_volatile_ok v (true);
+      expand_insn (code_for_aarch64_pred_mov (mode), 3, ops);
+    }
 }
 
 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cond_arith_6.c b/gcc/testsuite/gcc.target/aarch64/sve/cond_arith_6.c
index 4085ab1244450a6f7a856a13f9fde07b7d216f8c..d5a12f1df0777616506325fc7e73e0faf2cffa9b 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/sve/cond_arith_6.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cond_arith_6.c
@@ -8,7 +8,8 @@ f (float *x)
       x[i] -= 1.0f;
 }
 
-/* { dg-final { scan-assembler {\tld1w\tz} } } */
+/* { dg-final { scan-assembler {\tld1w\tz} { target aarch64_big_endian } } } */
+/* { dg-final { scan-assembler {\tldr\tq} { target aarch64_little_endian } } } */
 /* { dg-final { scan-assembler {\tfcmgt\tp} } } */
 /* { dg-final { scan-assembler {\tfsub\tz} } } */
 /* { dg-final { scan-assembler {\tst1w\tz} } } */

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/ldst_ptrue_128_to_neon.c b/gcc/testsuite/gcc.target/aarch64/sve/ldst_ptrue_128_to_neon.c
new file mode 100644 (file)
index 0000000..43d36e8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/ldst_ptrue_128_to_neon.c
@@ -0,0 +1,48 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -msve-vector-bits=128" } */
+/* { dg-require-effective-target aarch64_little_endian } */
+
+#include <arm_sve.h>
+
+#define TEST(TYPE, TY, B)                              \
+  sv##TYPE                                             \
+  ld1_##TY##B (TYPE *x)                                        \
+  {                                                    \
+    svbool_t pg = svptrue_b##B ();                     \
+    return svld1_##TY##B (pg, x);                      \
+  }                                                    \
+                                                       \
+  void                                                 \
+  st1_##TY##B (TYPE *x, sv##TYPE data)                 \
+  {                                                    \
+    svbool_t pg = svptrue_b##B ();                     \
+    svst1_##TY##B (pg, x, data);                       \
+  }                                                    \
+                                                       \
+  sv##TYPE                                             \
+  ld1_vol_##TY##B (volatile sv##TYPE *ptr)             \
+  {                                                    \
+    return *ptr;                                       \
+  }                                                    \
+                                                       \
+  void                                                 \
+  st1_vol_##TY##B (volatile sv##TYPE *ptr, sv##TYPE x) \
+  {                                                    \
+    *ptr = x;                                          \
+  }
+
+TEST (bfloat16_t, bf, 16)
+TEST (float16_t, f, 16)
+TEST (float32_t, f, 32)
+TEST (float64_t, f, 64)
+TEST (int8_t, s, 8)
+TEST (int16_t, s, 16)
+TEST (int32_t, s, 32)
+TEST (int64_t, s, 64)
+TEST (uint8_t, u, 8)
+TEST (uint16_t, u, 16)
+TEST (uint32_t, u, 32)
+TEST (uint64_t, u, 64)
+
+/* { dg-final { scan-assembler-times {\tldr\tq0, \[x0\]} 24 } } */
+/* { dg-final { scan-assembler-times {\tstr\tq0, \[x0\]} 24 } } */

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_4_128.c b/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_4_128.c
index 87d528c84cd307a6921df5aa627f2e962dda1c54..ac5f981490a36279c188ded863ca735be6a293e0 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_4_128.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_4_128.c
@@ -11,104 +11,91 @@
 
 /*
 ** callee_s8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s8, __SVInt8_t)
 
 /*
 ** callee_u8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u8, __SVUint8_t)
 
 /*
 ** callee_mf8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (mf8, __SVMfloat8_t)
 
 /*
 ** callee_s16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s16, __SVInt16_t)
 
 /*
 ** callee_u16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u16, __SVUint16_t)
 
 /*
 ** callee_f16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f16, __SVFloat16_t)
 
 /*
 ** callee_bf16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (bf16, __SVBfloat16_t)
 
 /*
 ** callee_s32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s32, __SVInt32_t)
 
 /*
 ** callee_u32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u32, __SVUint32_t)
 
 /*
 ** callee_f32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f32, __SVFloat32_t)
 
 /*
 ** callee_s64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s64, __SVInt64_t)
 
 /*
 ** callee_u64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u64, __SVUint64_t)
 
 /*
 ** callee_f64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f64, __SVFloat64_t)

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_5_128.c b/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_5_128.c
index 347a16c1367d1f93595ccdf277a43f73e3de5bcd..2fab6feb41cc22e0fa59f423bdc8eba2efbc5aa9 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_5_128.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pcs/return_5_128.c
@@ -13,104 +13,91 @@
 
 /*
 ** callee_s8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s8, svint8_t)
 
 /*
 ** callee_u8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u8, svuint8_t)
 
 /*
 ** callee_mf8:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (mf8, svmfloat8_t)
 
 /*
 ** callee_s16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s16, svint16_t)
 
 /*
 ** callee_u16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u16, svuint16_t)
 
 /*
 ** callee_f16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f16, svfloat16_t)
 
 /*
 ** callee_bf16:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1h    z0\.h, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (bf16, svbfloat16_t)
 
 /*
 ** callee_s32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s32, svint32_t)
 
 /*
 ** callee_u32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u32, svuint32_t)
 
 /*
 ** callee_f32:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1w    z0\.s, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f32, svfloat32_t)
 
 /*
 ** callee_s64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (s64, svint64_t)
 
 /*
 ** callee_u64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (u64, svuint64_t)
 
 /*
 ** callee_f64:
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 CALLEE (f64, svfloat64_t)

diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_128.c b/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_128.c
index d99ce1202a9200c834b90741d2958a379cb4f451..29bdaf3ae59b31452cd11a31577dcbc85de3058e 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_128.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pcs/struct_3_128.c
@@ -473,17 +473,16 @@ SEL2 (struct, pst_uniform4)
 **     sub     sp, sp, #144
 **     add     (x[0-9]+), sp, #?31
 **     and     x7, \1, #?(?:-32|4294967264)
-**     ptrue   (p[0-7])\.b, vl16
-**     st1w    z0\.s, \2, \[x7\]
-**     add     (x[0-9]+), x7, #?32
+**     mov     (x[0-9]+), x7
+**     str     q0, \[\2\], 32
 ** (
-**     str     z1, \[\3\]
-**     str     z2, \[\3, #1, mul vl\]
+**     str     z1, \[\2\]
+**     str     z2, \[\2, #1, mul vl\]
 ** |
-**     stp     q1, q2, \[\3\]
+**     stp     q1, q2, \[\2\]
 ** )
-**     str     z3, \[\3, #2, mul vl\]
-**     st1w    z4\.s, \2, \[x7, #6, mul vl\]
+**     str     z3, \[\2, #2, mul vl\]
+**     str     q4, \[x7, 96\]
 **     add     sp, sp, #?144
 **     ret
 */
@@ -516,20 +515,12 @@ SEL2 (struct, pst_mixed1)
 ** test_pst_mixed1:
 **     sub     sp, sp, #176
 **     str     p0, \[sp\]
-**     ptrue   p0\.b, vl16
-**     st1h    z0\.h, p0, \[sp, #1, mul vl\]
-**     st1h    z1\.h, p0, \[sp, #2, mul vl\]
-**     st1w    z2\.s, p0, \[sp, #3, mul vl\]
-**     st1d    z3\.d, p0, \[sp, #4, mul vl\]
+**     stp     q0, q1, \[sp, 16\]
+**     stp     q2, q3, \[sp, 48\]
 **     str     p1, \[sp, #40, mul vl\]
 **     str     p2, \[sp, #41, mul vl\]
-**     st1b    z4\.b, p0, \[sp, #6, mul vl\]
-**     st1h    z5\.h, p0, \[sp, #7, mul vl\]
-**     ...
-**     st1w    z6\.s, p0, [^\n]*
-**     ...
-**     st1d    z7\.d, p0, [^\n]*
-**     ...
+**     stp     q4, q5, \[sp, 96\]
+**     stp     q6, q7, \[sp, 128\]
 **     str     p3, \[sp, #80, mul vl\]
 **     mov     (x7, sp|w7, wsp)
 **     add     sp, sp, #?176
@@ -557,15 +548,11 @@ SEL2 (struct, pst_mixed2)
 ** test_pst_mixed2:
 **     sub     sp, sp, #128
 **     str     p0, \[sp\]
-**     ptrue   (p[03])\.b, vl16
-**     add     (x[0-9]+), sp, #?2
-**     st1b    z0\.b, \1, \[\2\]
+**     str     q0, \[sp, 2\]
 **     str     p1, \[sp, #9, mul vl\]
-**     add     (x[0-9]+), sp, #?20
-**     st1b    z1\.b, \1, \[\3\]
+**     str     q1, \[sp, 20\]
 **     str     p2, \[sp, #18, mul vl\]
-**     add     (x[0-9]+), sp, #?38
-**     st1b    z2\.b, \1, \[\4\]
+**     str     q2, \[sp, 38\]
 ** (
 **     str     z3, \[sp, #4, mul vl\]
 **     str     z4, \[sp, #5, mul vl\]
@@ -595,8 +582,7 @@ SEL2 (struct, pst_big1)
 
 /*
 ** test_pst_big1_a: { target lp64 }
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 /*
@@ -760,8 +746,7 @@ test_pst_big3_d (struct pst_big3 x)
 
 /*
 ** test_pst_big3_e: { target lp64 }
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0, #1, mul vl\]
+**     ldr     q0, \[x0, 16\]
 **     ret
 */
 /*
@@ -780,8 +765,7 @@ test_pst_big3_e (struct pst_big3 x)
 
 /*
 ** test_pst_big3_f: { target lp64 }
-**     ptrue   (p[0-7])\.b, vl16
-**     ld1b    z0\.b, \1/z, \[x0, #5, mul vl\]
+**     ldr     q0, \[x0, 80\]
 **     ret
 */
 /*
@@ -1035,8 +1019,7 @@ SEL2 (struct, nonpst6)
 
 /*
 ** test_nonpst6: { target lp64 }
-**     ptrue   (p[0-3])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 /*
@@ -1063,8 +1046,7 @@ SEL2 (struct, nonpst7)
 
 /*
 ** test_nonpst7: { target lp64 }
-**     ptrue   (p[0-3])\.b, vl16
-**     ld1d    z0\.d, \1/z, \[x0\]
+**     ldr     q0, \[x0\]
 **     ret
 */
 /*