The following example
int foo2 (char *buf, int len) {
int x;
for (int i =0; i < len; i++) {
x += (int) i * buf[i];
}
return x;
}
compiled with -O3 -mcpu=neoverse-v2 used to generate a 4x unrolled MLA sequence
mla z29.s, p7/m, z2.s, z0.s
mla z27.s, p7/m, z4.s, z26.s
mla z30.s, p7/m, z1.s, z0.s
mla z28.s, p7/m, z23.s, z3.s
but now generates MUL + ADD
mul z2.s, z2.s, z1.s
mul z4.s, z4.s, z26.s
mul z1.s, z24.s, z1.s
mul z3.s, z23.s, z3.s
add z29.s, z2.s, z29.s
add z30.s, z1.s, z30.s
add z28.s, z3.s, z28.s
add z0.s, z4.s, z0.s
This is since the fix for
r16-3328-g3182e95eda4 we now insert casts around the
reduction addend. This causes convert_mult_to_fma to miss the mul + add
sequence.
This patch teaches it to look around the casts for the operands and only accept
the conversions if it's essentially only a sign changing operations.
Concretely, it converts:
# vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)>
vect__3.8_38 = MEM <vector([4,4]) char> [(char *)_16];
vect__4.12_45 = (vector([4,4]) int) vect__3.8_38;
vect__5.14_54 = vect__4.12_45 * vect_vec_iv_.13_49;
vect_x_12.17_62 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect__5.14_54);
vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58);
vect_x_12.17_64 = vect_x_12.17_62 + vect_x_12.17_63;
vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) int>(vect_x_12.17_64);
into:
# vect_vec_iv_.13_49 = PHI <_50(5), { 0, 1, 2, ... }(4)>
vect__3.8_38 = MEM <vector([4,4]) charD.8> [(charD.8 *)_16];
vect__4.12_45 = (vector([4,4]) intD.7) vect__3.8_38;
vect_x_12.17_63 = VIEW_CONVERT_EXPR<vector([4,4]) unsigned int>(vect_x_16.15_58);
_2 = (vector([4,4]) unsigned int) vect_vec_iv_.13_49;
_1 = (vector([4,4]) unsigned int) vect__4.12_45;
vect_x_12.17_64 = .FMA (_1, _2, vect_x_12.17_63);
vect_x_12.16_65 = VIEW_CONVERT_EXPR<vector([4,4]) intD.7>(vect_x_12.17_64);
thus restoring FMAs on reductions.
gcc/ChangeLog:
PR tree-optimization/122749
* tree-ssa-math-opts.cc (convert_mult_to_fma_1, convert_mult_to_fma):
Unwrap converts around addend.
gcc/testsuite/ChangeLog:
PR tree-optimization/122749
* gcc.target/aarch64/pr122749_1.c: New test.
* gcc.target/aarch64/pr122749_2.c: New test.
* gcc.target/aarch64/pr122749_3.c: New test.
* gcc.target/aarch64/pr122749_4.c: New test.
* gcc.target/aarch64/pr122749_5.c: New test.
* gcc.target/aarch64/pr122749_6.c: New test.
* gcc.target/aarch64/pr122749_8.c: New test.
* gcc.target/aarch64/pr122749_9.c: New test.
* gcc.target/aarch64/sve/pr122749_1.c: New test.
* gcc.target/aarch64/sve/pr122749_11.c: New test.
* gcc.target/aarch64/sve/pr122749_12.c: New test.
* gcc.target/aarch64/sve/pr122749_13.c: New test.
* gcc.target/aarch64/sve/pr122749_14.c: New test.
* gcc.target/aarch64/sve/pr122749_2.c: New test.
* gcc.target/aarch64/sve/pr122749_3.c: New test.
* gcc.target/aarch64/sve/pr122749_4.c: New test.
* gcc.target/aarch64/sve/pr122749_5.c: New test.
* gcc.target/aarch64/sve/pr122749_6.c: New test.
* gcc.target/aarch64/sve/pr122749_8.c: New test.
* gcc.target/aarch64/sve/pr122749_9.c: New test.
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 2 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.FMA" 4 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target arm_v8_neon_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 --param vect-epilogues-nomask=0 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\.FMA" 1 "widening_mul" { xfail *-*-* } } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT8_MAX, INT8_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint8_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT8_MAX, 7, 0, UINT8_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT16_MAX, 7, 0, UINT16_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT32_MAX, 7, 0, UINT32_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef uint64_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, 2, UINT64_MAX, 7, 0, UINT64_MAX, 5, 9 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int16_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT16_MAX, INT16_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int32_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT32_MAX, INT32_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+#include <limits.h>
+#include <stdint.h>
+
+typedef int64_t elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1, -2, INT64_MAX, INT64_MIN, 5, -7, 3, -4 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef float elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0f, 2.0f, 1.0f, 2.0f, 1.0f, 2.0f };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-additional-options "-Ofast -std=gnu99 -fwrapv -fdump-tree-vect-details -fdump-tree-widening_mul" } */
+
+typedef double elem_t;
+
+__attribute__ ((noipa))
+elem_t
+foo2 (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+static elem_t
+reference (elem_t *buf, int len)
+{
+ elem_t x = 0;
+
+#pragma GCC novector
+ for (int i = 0; i < len; i++)
+ x += (elem_t) i * buf[i];
+
+ return x;
+}
+
+int
+main (void)
+{
+ elem_t buf[] = { 1.0, 2.0, 1.0, 2.0, 1.0, 2.0 };
+ int len = sizeof (buf) / sizeof (buf[0]);
+ elem_t want = reference (buf, len);
+ elem_t got = foo2 (buf, len);
+
+ if (want != got)
+ __builtin_abort ();
+
+ return 0;
+}
+
+/* { dg-final { scan-tree-dump-times "\\.COND_FMA" 1 "widening_mul" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" } } */
if (is_gimple_debug (use_stmt))
continue;
+ /* If the use is a type convert, look further into it if the operations
+ are the same under two's complement. */
+ tree lhs_type;
+ if (gimple_assign_cast_p (use_stmt)
+ && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
+ && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
+ {
+ tree cast_lhs = gimple_get_lhs (use_stmt);
+ gimple *tmp_use;
+ use_operand_p tmp_use_p;
+ if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
+ {
+ release_defs (use_stmt);
+ use_stmt = tmp_use;
+ result = cast_lhs;
+ gsi_remove (&gsi, true);
+ gsi = gsi_for_stmt (use_stmt);
+ }
+ }
+
if (is_gimple_assign (use_stmt)
&& gimple_assign_rhs_code (use_stmt) == NEGATE_EXPR)
{
if (seq)
gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
+ /* Ensure all the operands are of the same type. Use the type of the
+ addend as that's the statement being replaced. */
+ op2 = gimple_convert (&gsi, true, GSI_SAME_STMT,
+ UNKNOWN_LOCATION, TREE_TYPE (addop), op2);
+ mulop1 = gimple_convert (&gsi, true, GSI_SAME_STMT,
+ UNKNOWN_LOCATION, TREE_TYPE (addop), mulop1);
+
if (len)
fma_stmt
= gimple_build_call_internal (IFN_COND_LEN_FMA, 7, cond, mulop1, op2,
if (is_gimple_debug (use_stmt))
continue;
+ /* If the use is a type convert, look further into it if the operations
+ are the same under two's complement. */
+ tree lhs_type;
+ if (gimple_assign_cast_p (use_stmt)
+ && (lhs_type = TREE_TYPE (gimple_get_lhs (use_stmt)))
+ && tree_nop_conversion_p (lhs_type, TREE_TYPE (op1)))
+ {
+ tree cast_lhs = gimple_get_lhs (use_stmt);
+ gimple *tmp_use;
+ use_operand_p tmp_use_p;
+ if (single_imm_use (cast_lhs, &tmp_use_p, &tmp_use))
+ use_stmt = tmp_use;
+ }
+
/* For now restrict this operations to single basic blocks. In theory
we would want to support sinking the multiplication in
m = a*b;
projects / thirdparty / gcc.git / commitdiff
