i386: Fix Win64 __int128 return handling [PR78799]

GCC could return __int128 values in SSE (%xmm0) on Windows x64 instead
of following the MS x64 ABI. Root cause: return classification allowed
128-bit integer-like scalars to be treated as direct register returns.
This patch updates the Windows x64 return-classification and codegen to
treat int128 as an indirect return (caller-provided slot passed as first
argument, pointer returned in RAX).

gcc:
	PR target/78799
	* config/i386/i386.cc (function_value_ms_64): Do not treat
	integral 16-byte values as SSE returns.
	(ix86_return_in_memory): Likewise avoid treating integral 16-byte
	values as XMM returns.

gcc/testsuite:
	* gcc.target/i386/pr78799.c: New test.

Signed-off-by: Oleg Tolmatcev <oleg.tolmatcev@gmail.com>
Signed-off-by: Jonathan Yong <10walls@gmail.com>

diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index 23f65ab401d1e6c97f5995087a33673ec2561de5..31a057e81cd959bc20d3c0f4f6bbb1668a6a105e 100644 (file)
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -4348,10 +4348,9 @@ function_value_ms_64 (machine_mode orig_mode, machine_mode mode,
            break;
          if (valtype != NULL_TREE
              && !VECTOR_INTEGER_TYPE_P (valtype)
-             && !INTEGRAL_TYPE_P (valtype)
              && !VECTOR_FLOAT_TYPE_P (valtype))
            break;
-         if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
+         if (VECTOR_MODE_P (mode)
              && !COMPLEX_MODE_P (mode))
            regno = FIRST_SSE_REG;
          break;
@@ -4458,9 +4457,8 @@ ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
          /* __m128 is returned in xmm0.  256/512-bit vector values are
             returned in ymm0/zmm0 when AVX/AVX512 is enabled.  */
          if ((!type || VECTOR_INTEGER_TYPE_P (type)
-              || INTEGRAL_TYPE_P (type)
               || VECTOR_FLOAT_TYPE_P (type))
-             && (SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
+             && VECTOR_MODE_P (mode)
              && !COMPLEX_MODE_P (mode)
              && ((GET_MODE_SIZE (mode) == 16 || size == 16)
                  || (TARGET_AVX && (GET_MODE_SIZE (mode) == 32 || size == 32))

diff --git a/gcc/testsuite/gcc.target/i386/pr78799.c b/gcc/testsuite/gcc.target/i386/pr78799.c
new file mode 100644 (file)
index 0000000..fcd6fe3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/pr78799.c
@@ -0,0 +1,15 @@
+/* { dg-do compile { target { x86_64-*-mingw* } } } */
+/* { dg-options "-O2" } */
+
+/* PR78799: verify Win64 __int128 return uses an indirect sret-style path:
+   RCX points to the return slot, RDX points to the argument object.  */
+__attribute__((ms_abi, noinline, noclone))
+__int128
+ret_i128 (__int128 a)
+{
+  return a;
+}
+
+/* { dg-final { scan-assembler "movdqa\t\\(%rdx\\), %xmm0" } } */
+/* { dg-final { scan-assembler "movq\t%rcx, %rax" } } */
+/* { dg-final { scan-assembler "movaps\t%xmm0, \\(%rcx\\)" } } */