tree-ssa-dce: Punt on allocations with too large constant sizes [PR118224]

As suggested by Richi in the PR, the following patch will fail to DCE
allocation calls if they have constant size which is too large (over
PTRDIFF_MAX), or for the case of calloc, if either of the arguments
is too large (in that case in theory the call could succeed if the other
argument is variable zero but who cares) or if both are constant and
their product overflows or is above PTRDIFF_MAX.

This will make some pedantic conformance tests happy, though if one
hides the size one will still need to use -fno-malloc-dce or obfuscate even
the malloc etc. uses.  If the size is constant and too large, it isn't worth
trying to optimize it.

2025-01-06  Jakub Jelinek  <jakub@redhat.com>

	PR tree-optimization/118224
	* tree-ssa-dce.cc (is_removable_allocation_p): Don't return true
	for allocations with constant size argument larger than PTRDIFF_MAX
	or for calloc with one of the arguments constant larger than
	PTRDIFF_MAX or their product known constant above PTRDIFF_MAX.
	Fix comment typos, furhter -> further and then -> than.
	* lto-section-in.cc (lto_free_function_in_decl_state_for_node):
	Fix comment typo, furhter -> further.

	* gcc.dg/pr118224.c: New test.
	* c-c++-common/ubsan/vla-1.c (bar): Use noipa attribute instead
	of noinline, noclone.

diff --git a/gcc/lto-section-in.cc b/gcc/lto-section-in.cc
index 0068aadb142ca6c308f1214ec795e5042490b31c..1dd9520137a17f673c6e67562e0ae7adf1b5d6ba 100644 (file)
--- a/gcc/lto-section-in.cc
+++ b/gcc/lto-section-in.cc
@@ -428,7 +428,7 @@ lto_free_function_in_decl_state (struct lto_in_decl_state *state)
   ggc_free (state);
 }
 
-/* Free decl_states associated with NODE.  This makes it possible to furhter
+/* Free decl_states associated with NODE.  This makes it possible to further
    release trees needed by the NODE's body.  */
 
 void

diff --git a/gcc/testsuite/c-c++-common/ubsan/vla-1.c b/gcc/testsuite/c-c++-common/ubsan/vla-1.c
index c97465edae195442a71ee66ab25015a2ac4fc8fc..b29d904dea393f1fb3f0e23f8afed76651aaa1d1 100644 (file)
--- a/gcc/testsuite/c-c++-common/ubsan/vla-1.c
+++ b/gcc/testsuite/c-c++-common/ubsan/vla-1.c
@@ -6,7 +6,7 @@ int x = -1;
 double di = -3.2;
 V v = -6;
 
-static int __attribute__ ((noinline, noclone))
+static int __attribute__ ((noipa))
 bar (void)
 {
   return -4;

diff --git a/gcc/testsuite/gcc.dg/pr118224.c b/gcc/testsuite/gcc.dg/pr118224.c
new file mode 100644 (file)
index 0000000..683f4dc
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr118224.c
@@ -0,0 +1,31 @@
+/* PR tree-optimization/118224 */
+/* { dg-do run } */
+/* { dg-options "-O2 -w" } */
+
+#include <stdlib.h>
+
+void
+foo (__SIZE_TYPE__ s)
+{
+  if (__builtin_calloc (s, ~(__SIZE_TYPE__) 0))
+    __builtin_abort ();
+  if (__builtin_calloc (~(__SIZE_TYPE__) 0, s))
+    __builtin_abort ();
+}
+
+int
+main ()
+{
+  if (__builtin_malloc (~(__SIZE_TYPE__) 0))
+    __builtin_abort ();
+#ifdef __GLIBC_PREREQ
+#if __GLIBC_PREREQ (2, 16)
+  /* aligned_alloc was added in glibc 2.16 */
+  if (__builtin_aligned_alloc (32, ~(__SIZE_TYPE__) 0))
+    __builtin_abort ();
+#endif
+#endif
+  if (__builtin_calloc ((~(__SIZE_TYPE__) 0) / 2, 3))
+    __builtin_abort ();
+  foo (1);
+}

diff --git a/gcc/tree-ssa-dce.cc b/gcc/tree-ssa-dce.cc
index 11f7e7472efa603c49ed17f62f437dd3b6555d48..adc7941033176ef431e804c83b9ed5b5be337675 100644 (file)
--- a/gcc/tree-ssa-dce.cc
+++ b/gcc/tree-ssa-dce.cc
@@ -243,38 +243,101 @@ mark_operand_necessary (tree op)
 
 /* Return true if STMT is a call to allocation function that can be
    optimized out if the memory block is never used for anything else
-   then NULL pointer check or free.
-   If NON_NULL_CHECK is false, we can furhter assume that return value
-   is never checked to be non-NULL. */
+   than NULL pointer check or free.
+   If NON_NULL_CHECK is false, we can further assume that return value
+   is never checked to be non-NULL.
+   Don't return true if it is called with constant size (or sizes for calloc)
+   and the size is excessively large (larger than PTRDIFF_MAX, for calloc
+   either argument larger than PTRDIFF_MAX or both constant and their product
+   larger than PTRDIFF_MAX).  */
 
 static bool
 is_removable_allocation_p (gcall *stmt, bool non_null_check)
 {
-  tree callee = gimple_call_fndecl (stmt);
+  int arg = -1;
+  tree callee = gimple_call_fndecl (stmt), a1, a2;
   if (callee != NULL_TREE
       && fndecl_built_in_p (callee, BUILT_IN_NORMAL))
     switch (DECL_FUNCTION_CODE (callee))
       {
       case BUILT_IN_MALLOC:
+       arg = 1;
+       goto do_malloc;
       case BUILT_IN_ALIGNED_ALLOC:
+       arg = 2;
+       goto do_malloc;
       case BUILT_IN_CALLOC:
+       arg = 3;
+       goto do_malloc;
       CASE_BUILT_IN_ALLOCA:
+       arg = 1;
+       goto do_malloc;
       case BUILT_IN_STRDUP:
       case BUILT_IN_STRNDUP:
-       return non_null_check ? flag_malloc_dce > 1 : flag_malloc_dce;
+       arg = 0;
+       /* FALLTHRU */
+      do_malloc:
+       if (non_null_check)
+         {
+           if (flag_malloc_dce <= 1)
+             return false;
+         }
+       else if (!flag_malloc_dce)
+         return false;
+       break;
 
       case BUILT_IN_GOMP_ALLOC:
-       return true;
+       arg = 2;
+       break;
 
       default:;
       }
 
-  if (callee != NULL_TREE
+  if (arg == -1
+      && callee != NULL_TREE
       && flag_allocation_dce
       && gimple_call_from_new_or_delete (stmt)
       && DECL_IS_REPLACEABLE_OPERATOR_NEW_P (callee))
-    return true;
-  return false;
+    arg = 1;
+
+  switch (arg)
+    {
+    case -1:
+      return false;
+    case 0:
+      return true;
+    case 1:
+    case 2:
+      if (gimple_call_num_args (stmt) < (unsigned) arg)
+       return false;
+      a1 = gimple_call_arg (stmt, arg - 1);
+      if (tree_fits_uhwi_p (a1)
+         && (tree_to_uhwi (a1)
+             > tree_to_uhwi (TYPE_MAX_VALUE (ptrdiff_type_node))))
+       return false;
+      return true;
+    case 3:
+      if (gimple_call_num_args (stmt) < 2)
+       return false;
+      a1 = gimple_call_arg (stmt, 0);
+      a2 = gimple_call_arg (stmt, 1);
+      if (tree_fits_uhwi_p (a1)
+         && (tree_to_uhwi (a1)
+             > tree_to_uhwi (TYPE_MAX_VALUE (ptrdiff_type_node))))
+       return false;
+      if (tree_fits_uhwi_p (a2)
+         && (tree_to_uhwi (a2)
+             > tree_to_uhwi (TYPE_MAX_VALUE (ptrdiff_type_node))))
+       return false;
+      if (TREE_CODE (a1) == INTEGER_CST
+         && TREE_CODE (a2) == INTEGER_CST
+         && (wi::to_widest (a1) + wi::to_widest (a2)
+             > tree_to_uhwi (TYPE_MAX_VALUE (ptrdiff_type_node))))
+       return false;
+      return true;
+    default:
+      gcc_unreachable ();
+    }
 }
 
 /* Return true if STMT is a conditional