Improve check against integer wraparound in hcreate_r [BZ #18240]

(cherry picked from commit bae7c7c764413b23e61cb099ce33be4c4ee259bb)

diff --git a/ChangeLog b/ChangeLog
index e8189959698adee6df417a42065dc272548cf3bb..ed20b9b4b5039892850f24929308bac5623b9555 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+2016-01-27  Paul Eggert  <eggert@cs.ucla.edu>
+
+       [BZ #18240]
+       * misc/hsearch_r.c (isprime, __hcreate_r): Protect against
+       unsigned int wraparound.
+
+2016-01-27  Florian Weimer  <fweimer@redhat.com>
+
+       [BZ #18240]
+       * misc/bug18240.c: New test.
+       * misc/Makefile (tests): Add it.
+
 2015-08-25  Ondřej Bílka  <neleai@seznam.cz>
 
        [BZ #18240]

diff --git a/NEWS b/NEWS
index 99e68d20dee1b1df8a6c14d66a171af5eb6ae71d..d1daf9bec9c5e3011ed25b633bd4dc0367343259 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -9,8 +9,8 @@ Version 2.22.1
 
 * The following bugs are resolved with this release:
 
-  17905, 18421, 18480, 18589, 18743, 18778, 18781, 18787, 18796, 18870,
-  18887, 18921, 18928, 18969, 18985, 19018, 19058, 19174, 19178.
+  17905, 18420, 18421, 18480, 18589, 18743, 18778, 18781, 18787, 18796,
+  18870, 18887, 18921, 18928, 18969, 18985, 19018, 19058, 19174, 19178.
 
 * The LD_POINTER_GUARD environment variable can no longer be used to
   disable the pointer guard feature.  It is always enabled.

diff --git a/misc/Makefile b/misc/Makefile
index 2f5edf6316dacc282def12ed3363432b974bc485..12055cebad5e9182d5a80ce39229b385d5eab624 100644 (file)
--- a/misc/Makefile
+++ b/misc/Makefile
@@ -77,7 +77,7 @@ gpl2lgpl := error.c error.h
 
 tests := tst-dirname tst-tsearch tst-fdset tst-efgcvt tst-mntent tst-hsearch \
         tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1 \
-        tst-mntent-blank-corrupt tst-mntent-blank-passno
+        tst-mntent-blank-corrupt tst-mntent-blank-passno bug18240
 ifeq ($(run-built-tests),yes)
 tests-special += $(objpfx)tst-error1-mem.out
 endif

diff --git a/misc/bug18240.c b/misc/bug18240.c
new file mode 100644 (file)
index 0000000..4b26865
--- /dev/null
+++ b/misc/bug18240.c
@@ -0,0 +1,75 @@
+/* Test integer wraparound in hcreate.
+   Copyright (C) 2016 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <limits.h>
+#include <search.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static void
+test_size (size_t size)
+{
+  int res = hcreate (size);
+  if (res == 0)
+    {
+      if (errno == ENOMEM)
+        return;
+      printf ("error: hcreate (%zu): %m\n", size);
+      exit (1);
+    }
+  char *keys[100];
+  for (int i = 0; i < 100; ++i)
+    {
+      if (asprintf (keys + i, "%d", i) < 0)
+        {
+          printf ("error: asprintf: %m\n");
+          exit (1);
+        }
+      ENTRY e = { keys[i], (char *) "value" };
+      if (hsearch (e, ENTER) == NULL)
+        {
+          printf ("error: hsearch (\"%s\"): %m\n", keys[i]);
+          exit (1);
+        }
+    }
+  hdestroy ();
+
+  for (int i = 0; i < 100; ++i)
+    free (keys[i]);
+}
+
+static int
+do_test (void)
+{
+  test_size (500);
+  test_size (-1);
+  test_size (-3);
+  test_size (INT_MAX - 2);
+  test_size (INT_MAX - 1);
+  test_size (INT_MAX);
+  test_size (((unsigned) INT_MAX) + 1);
+  test_size (UINT_MAX - 2);
+  test_size (UINT_MAX - 1);
+  test_size (UINT_MAX);
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"

diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index 559df29cf7b124dbdf88f089f1ed5a784b364105..661f0f6df13aa51f8adcba074d0ccb9821d7a2d7 100644 (file)
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -46,15 +46,12 @@ static int
 isprime (unsigned int number)
 {
   /* no even number will be passed */
-  unsigned int div = 3;
-
-  while (div * div < number && number % div != 0)
-    div += 2;
-
-  return number % div != 0;
+  for (unsigned int div = 3; div <= number / div; div += 2)
+    if (number % div == 0)
+      return 0;
+  return 1;
 }
 
-
 /* Before using the hash table we must allocate memory for it.
    Test for an existing table are done. We allocate one element
    more as the found prime number says. This is done for more effective
@@ -73,13 +70,6 @@ __hcreate_r (nel, htab)
       return 0;
     }
 
-  if (nel >= SIZE_MAX / sizeof (_ENTRY))
-    {
-      __set_errno (ENOMEM);
-      return 0;
-    }
-
-
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;
@@ -88,10 +78,19 @@ __hcreate_r (nel, htab)
      use will not work.  */
   if (nel < 3)
     nel = 3;
-  /* Change nel to the first prime number not smaller as nel. */
-  nel |= 1;      /* make odd */
-  while (!isprime (nel))
-    nel += 2;
+
+  /* Change nel to the first prime number in the range [nel, UINT_MAX - 2],
+     The '- 2' means 'nel += 2' cannot overflow.  */
+  for (nel |= 1; ; nel += 2)
+    {
+      if (UINT_MAX - 2 < nel)
+       {
+         __set_errno (ENOMEM);
+         return 0;
+       }
+      if (isprime (nel))
+       break;
+    }
 
   htab->size = nel;
   htab->filled = 0;