dynarray: Set errno on overflow-induced allocation failure

This allows the caller to return directly on such an error, with an
appropriate errno value.

diff --git a/ChangeLog b/ChangeLog
index b0f01c2207697e32a552bdb9b897387ab7b10b56..1d8bcde330a51e1b3697ead746bc91eb91cf16ff 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2017-08-30  Florian Weimer  <fweimer@redhat.com>
+
+       * malloc/dynarray_emplace_enlarge.c
+       (__libc_dynarray_emplace_enlarge): Set errno on overflow.
+       * malloc/dynarray_resize.c (__libc_dynarray_resize): Likewise.
+       * malloc/tst-dynarray.c (test_long_overflow): New function.
+       (do_test): Call it.
+
 2017-08-30  Florian Weimer  <fweimer@redhat.com>
 
        * malloc/malloc.c (ARENA_CORRUPTION_BIT, arena_is_corrupt)

diff --git a/malloc/dynarray_emplace_enlarge.c b/malloc/dynarray_emplace_enlarge.c
index dfc70017cec238000d67cc643b9d54907c8a4056..0fb032765cc999520aaf4a63a58633bc97b3bc3a 100644 (file)
--- a/malloc/dynarray_emplace_enlarge.c
+++ b/malloc/dynarray_emplace_enlarge.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <dynarray.h>
+#include <errno.h>
 #include <malloc-internal.h>
 #include <stdlib.h>
 #include <string.h>
@@ -43,8 +44,11 @@ __libc_dynarray_emplace_enlarge (struct dynarray_header *list,
     {
       new_allocated = list->allocated + list->allocated / 2 + 1;
       if (new_allocated <= list->allocated)
-        /* Overflow.  */
-        return false;
+        {
+          /* Overflow.  */
+          __set_errno (ENOMEM);
+          return false;
+        }
     }
 
   size_t new_size;

diff --git a/malloc/dynarray_resize.c b/malloc/dynarray_resize.c
index e6dc9fbc68587de4ae9b16792e6366933884f4ea..63c981bf61f671459d5a20c35f1893a38bf28afc 100644 (file)
--- a/malloc/dynarray_resize.c
+++ b/malloc/dynarray_resize.c
@@ -17,6 +17,7 @@
    <http://www.gnu.org/licenses/>.  */
 
 #include <dynarray.h>
+#include <errno.h>
 #include <malloc-internal.h>
 #include <stdlib.h>
 #include <string.h>
@@ -38,7 +39,11 @@ __libc_dynarray_resize (struct dynarray_header *list, size_t size,
 
   size_t new_size_bytes;
   if (check_mul_overflow_size_t (size, element_size, &new_size_bytes))
-    return false;
+    {
+      /* Overflow.  */
+      __set_errno (ENOMEM);
+      return false;
+    }
   void *new_array;
   if (list->array == scratch)
     {

diff --git a/malloc/tst-dynarray.c b/malloc/tst-dynarray.c
index 2206d75e318aaa3f616232d39805451bf2c6ad99..d11f7bb8a343a16afc4a3152a291864d71ed90b6 100644 (file)
--- a/malloc/tst-dynarray.c
+++ b/malloc/tst-dynarray.c
@@ -18,6 +18,9 @@
 
 #include "tst-dynarray-shared.h"
 
+#include <errno.h>
+#include <stdint.h>
+
 #define DYNARRAY_STRUCT dynarray_long
 #define DYNARRAY_ELEMENT long
 #define DYNARRAY_PREFIX dynarray_long_
@@ -463,6 +466,31 @@ test_long_init (void)
   }
 }
 
+/* Test overflow in resize.  */
+static void
+test_long_overflow (void)
+{
+  {
+    struct dynarray_long dyn;
+    dynarray_long_init (&dyn);
+    errno = EINVAL;
+    TEST_VERIFY (!dynarray_long_resize
+                 (&dyn, (SIZE_MAX / sizeof (long)) + 1));
+    TEST_VERIFY (errno == ENOMEM);
+    TEST_VERIFY (dynarray_long_has_failed (&dyn));
+  }
+
+  {
+    struct dynarray_long_noscratch dyn;
+    dynarray_long_noscratch_init (&dyn);
+    errno = EINVAL;
+    TEST_VERIFY (!dynarray_long_noscratch_resize
+                 (&dyn, (SIZE_MAX / sizeof (long)) + 1));
+    TEST_VERIFY (errno == ENOMEM);
+    TEST_VERIFY (dynarray_long_noscratch_has_failed (&dyn));
+  }
+}
+
 /* Test NUL-terminated string construction with the add function and
    the simple finalize function.  */
 static void
@@ -538,6 +566,7 @@ do_test (void)
   test_int ();
   test_str ();
   test_long_init ();
+  test_long_overflow ();
   test_zstr ();
   return 0;
 }