aarch64: Improve popcountti2 with SVE

The TImode popcount sequence can be slightly improved with SVE.
If we generate:
        ldr     q31, [x0]
        ptrue   p7.b, vl16
        cnt     z31.d, p7/m, z31.d
        addp    d31, v31.2d
        fmov    x0, d31
        ret

instead of:
h128:
        ldr     q31, [x0]
        cnt     v31.16b, v31.16b
        addv    b31, v31.16b
        fmov    w0, s31
        ret

we use the ADDP instruction for reduction, which is cheaper on all CPUs AFAIK,
as it is only a single 64-bit addition vs the tree of additions for ADDV.
For example, on a CPU like Grace we get a latency and throughput of 2,4 vs 4,1
for ADDV.
We do generate one more instruction due to the PTRUE being materialised, but that
is cheap itself and can be scheduled away from the critical path or even CSE'd
with other PTRUE constants.
As this sequence is larger code size-wise it is avoided for -Os.

Bootstrapped and tested on aarch64-none-linux-gnu.

Signed-off-by: Kyrylo Tkachov <ktkachov@nvidia.com>
gcc/

	* config/aarch64/aarch64.md (popcountti2): Add TARGET_SVE path.

gcc/testsuite/

	* gcc.target/aarch64/popcnt9.c: Add +nosve to target pragma.
	* gcc.target/aarch64/popcnt13.c: New test.

diff --git a/gcc/config/aarch64/aarch64.md b/gcc/config/aarch64/aarch64.md
index 509ef4c0f2f29e7c47fd6caf87a553946157575f..27efc9155dcb0fe2d57ff2cbedbf5cce75fa82ed 100644 (file)
--- a/gcc/config/aarch64/aarch64.md
+++ b/gcc/config/aarch64/aarch64.md
@@ -5771,6 +5771,19 @@
    (match_operand:TI 1 "register_operand")]
   "TARGET_SIMD && !TARGET_CSSC"
 {
+  /* For SVE we can do popcount on DImode chunks of the TImode argument
+     and then use a cheap ADDP reduction.  The SVE CNT instruction requires
+     materializing a PTRUE so don't do this if optimizing for size.  */
+  if (TARGET_SVE && !optimize_function_for_size_p (cfun))
+    {
+      rtx v = gen_reg_rtx (V2DImode);
+      rtx v1 = gen_reg_rtx (V2DImode);
+      emit_move_insn (v, gen_lowpart (V2DImode, operands[1]));
+      rtx p = aarch64_ptrue_reg (VNx2BImode, 16);
+      emit_insn (gen_aarch64_pred_popcountv2di (v1, p, v));
+      emit_insn (gen_reduc_plus_scal_v2di (operands[0], v1));
+      DONE;
+    }
   rtx v = gen_reg_rtx (V16QImode);
   rtx v1 = gen_reg_rtx (V16QImode);
   emit_move_insn (v, gen_lowpart (V16QImode, operands[1]));

diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt13.c b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
new file mode 100644 (file)
index 0000000..2a30e98
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt13.c
@@ -0,0 +1,24 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -fdump-tree-optimized" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#pragma GCC target "+nocssc+sve"
+
+/*
+** h128:
+**     ldr     q([0-9]+), \[x0\]
+**     ptrue   p([0-9]+).b, vl16
+**     cnt     z([0-9]+).d, p\2/m, z\1.d
+**     addp    d([0-9]+), v\3.2d
+**     fmov    x0, d\4
+**     ret
+*/
+
+unsigned h128 (const unsigned __int128 *a) {
+         return __builtin_popcountg (a[0]);
+}
+
+/* There should be only one POPCOUNT. */
+/* { dg-final { scan-tree-dump-times "POPCOUNT " 1 "optimized" } } */
+/* { dg-final { scan-tree-dump-not " __builtin_popcount"  "optimized" } } */
+

diff --git a/gcc/testsuite/gcc.target/aarch64/popcnt9.c b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
index c778fc7f420658c4db9824d3f309cf19507a725b..cfed8c58b7e0a2424fa98c35a41095adb36324fc 100644 (file)
--- a/gcc/testsuite/gcc.target/aarch64/popcnt9.c
+++ b/gcc/testsuite/gcc.target/aarch64/popcnt9.c
@@ -3,7 +3,7 @@
 /* { dg-final { check-function-bodies "**" "" } } */
 /* PR target/113042 */
 
-#pragma GCC target "+nocssc"
+#pragma GCC target "+nocssc+nosve"
 
 /*
 ** h128: