i386: return 256/512-bit vectors in registers for x86_64 MS ABI [PR89597]

On x86_64 Windows targets using MS ABI, GCC classified 256-bit and
512-bit vector returns as memory returns.  That caused hidden sret
pointer returns where YMM0/ZMM0 returns are expected.

Teach MS ABI return classification to keep 32-byte and 64-byte vector
returns in registers when AVX/AVX512F is enabled, matching the return
register selection path.

Also extend function_value_ms_64 so 32-byte and 64-byte eligible vector
returns are mapped to the SSE register class (YMM0/ZMM0 lanes).

Add tests for x86_64-*-mingw* that verify 256-bit and 512-bit vector
returns use YMM0/ZMM0 codegen.

gcc:

	PR target/89597
	* config/i386/i386.cc (function_value_ms_64): Handle 32-byte and
	64-byte vector returns in registers when supported.
	(ix86_return_in_memory): Do not force 32-byte/64-byte eligible
	vector returns to memory for MS ABI.

gcc/testsuite:

	* gcc.target/i386/pr89597-1.c: New test.
	* gcc.target/i386/pr89597-2.c: New test.

Signed-off-by: Oleg Tolmatcev <oleg.tolmatcev@gmail.com>
Signed-off-by: Jonathan Yong <10walls@gmail.com>

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 2744c74957802c27e577e06ec7dbc3df84a151df..a5559fe8a3300e1958eba4a7d9d03de4d9dbb335 100644 (file)
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -4346,9 +4346,17 @@ function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
 
   if (TARGET_SSE)
     {
-      switch (GET_MODE_SIZE (mode))
+      unsigned int mode_size = GET_MODE_SIZE (mode);
+
+      switch (mode_size)
        {
        case 16:
+       case 32:
+       case 64:
+         if (mode_size == 32 && !TARGET_AVX)
+           break;
+         if (mode_size == 64 && !TARGET_AVX512F)
+           break;
          if (valtype != NULL_TREE
              && !VECTOR_INTEGER_TYPE_P (valtype)
              && !INTEGRAL_TYPE_P (valtype)
@@ -4458,13 +4466,17 @@ ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
        {
          size = int_size_in_bytes (type);
 
-         /* __m128 is returned in xmm0.  */
+         /* __m128 is returned in xmm0.  256/512-bit vector values are
+            returned in ymm0/zmm0 when AVX/AVX512 is enabled.  */
          if ((!type || VECTOR_INTEGER_TYPE_P (type)
               || INTEGRAL_TYPE_P (type)
               || VECTOR_FLOAT_TYPE_P (type))
              && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
              && !COMPLEX_MODE_P (mode)
-             && (GET_MODE_SIZE (mode) == 16 || size == 16))
+             && ((GET_MODE_SIZE (mode) == 16 || size == 16)
+                 || (TARGET_AVX && (GET_MODE_SIZE (mode) == 32 || size == 32))
+                 || (TARGET_AVX512F
+                     && (GET_MODE_SIZE (mode) == 64 || size == 64))))
            return false;
 
          /* Otherwise, the size must be exactly in [1248]. */

diff --git a/gcc/testsuite/gcc.target/i386/pr89597-1.c b/gcc/testsuite/gcc.target/i386/pr89597-1.c
new file mode 100644 (file)
index 0000000..5dfd9cd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr89597-1.c
@@ -0,0 +1,12 @@
+/* { dg-do compile { target x86_64-*-mingw* } } */
+/* { dg-options "-O1 -mavx" } */
+
+typedef long long __m256i __attribute__ ((__vector_size__ (32)));
+
+__m256i
+foo (void)
+{
+  return (__m256i) { 1, 2, 3, 4 };
+}
+
+/* { dg-final { scan-assembler "ymm0" } } */

diff --git a/gcc/testsuite/gcc.target/i386/pr89597-2.c b/gcc/testsuite/gcc.target/i386/pr89597-2.c
new file mode 100644 (file)
index 0000000..25e737a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr89597-2.c
@@ -0,0 +1,12 @@
+/* { dg-do compile { target x86_64-*-mingw* } } */
+/* { dg-options "-O1 -mavx512f" } */
+
+typedef long long __m512i __attribute__ ((__vector_size__ (64)));
+
+__m512i
+foo (void)
+{
+  return (__m512i) { 1, 2, 3, 4, 5, 6, 7, 8 };
+}
+
+/* { dg-final { scan-assembler "zmm0" } } */