hcreate, hcreate_r should fail with ENOMEM if element count is too large (BZ18240...

diff --git a/README.google b/README.google
index 2d28310602ae1e5caf2c09978afd46e2e2f6e800..6cd12164bfc54d9127f8c5ec0bb011e6a202faba 100644 (file)
--- a/README.google
+++ b/README.google
@@ -641,3 +641,11 @@ sysdeps/generic/ldsodefs.h
   Always enable pointer guard (BZ18928, CVE-2015-8777)
   https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=a014cecd82b71b70a6a843e250e06b541ad524f7
   (stanshebs, backport)
+
+misc/Makefile
+misc/bug18240.c
+misc/hsearch_r.c
+  hcreate, hcreate_r should fail with ENOMEM if element count is too large (BZ18240, CVE-2015-8778)
+  https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=b5cba5cff937e5336ff23380785da80cab09146c
+  https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=3c9e8d9477aba0f514171bb4706670052544479b
+  (stanshebs, backport)

diff --git a/misc/Makefile b/misc/Makefile
index b039182fa08173ce9afec98fc74c860fc1cf84fc..ad9e92137a94a6a19614a1565c6bf69df1e4ee7c 100644 (file)
--- a/misc/Makefile
+++ b/misc/Makefile
@@ -76,7 +76,8 @@ install-lib := libg.a
 gpl2lgpl := error.c error.h
 
 tests := tst-dirname tst-tsearch tst-fdset tst-efgcvt tst-mntent tst-hsearch \
-        tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1
+        tst-error1 tst-pselect tst-insremque tst-mntent2 bug-hsearch1 \
+        bug18240
 ifeq ($(run-built-tests),yes)
 tests: $(objpfx)tst-error1-mem
 endif

diff --git a/misc/bug18240.c b/misc/bug18240.c
new file mode 100644 (file)
index 0000000..41d80a0
--- /dev/null
+++ b/misc/bug18240.c
@@ -0,0 +1,97 @@
+/* Test integer wraparound in hcreate.
+   Copyright (C) 2016-2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <errno.h>
+#include <limits.h>
+#include <search.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/resource.h>
+
+static void
+test_size (size_t size)
+{
+  int res = hcreate (size);
+  if (res == 0)
+    {
+      if (errno == ENOMEM)
+        return;
+      printf ("error: hcreate (%zu): %m\n", size);
+      exit (1);
+    }
+  char *keys[100];
+  for (int i = 0; i < 100; ++i)
+    {
+      if (asprintf (keys + i, "%d", i) < 0)
+        {
+          printf ("error: asprintf: %m\n");
+          exit (1);
+        }
+      ENTRY e = { keys[i], (char *) "value" };
+      if (hsearch (e, ENTER) == NULL)
+        {
+          printf ("error: hsearch (\"%s\"): %m\n", keys[i]);
+          exit (1);
+        }
+    }
+  hdestroy ();
+
+  for (int i = 0; i < 100; ++i)
+    free (keys[i]);
+}
+
+static int
+do_test (void)
+{
+  /* Limit the size of the process, so that memory allocation will
+     fail without impacting the entire system.  */
+  {
+    struct rlimit limit;
+    if (getrlimit (RLIMIT_AS, &limit) != 0)
+      {
+        printf ("getrlimit (RLIMIT_AS) failed: %m\n");
+        return 1;
+      }
+    long target = 100 * 1024 * 1024;
+    if (limit.rlim_cur == RLIM_INFINITY || limit.rlim_cur > target)
+      {
+        limit.rlim_cur = target;
+        if (setrlimit (RLIMIT_AS, &limit) != 0)
+          {
+            printf ("setrlimit (RLIMIT_AS) failed: %m\n");
+            return 1;
+          }
+      }
+  }
+
+  test_size (500);
+  test_size (-1);
+  test_size (-3);
+  test_size (INT_MAX - 2);
+  test_size (INT_MAX - 1);
+  test_size (INT_MAX);
+  test_size (((unsigned) INT_MAX) + 1);
+  test_size (UINT_MAX - 2);
+  test_size (UINT_MAX - 1);
+  test_size (UINT_MAX);
+  return 0;
+}
+
+#define TEST_FUNCTION do_test ()
+#include "../test-skeleton.c"

diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index 81c27d800c0d69896396823e0a387be6c3c6c2e7..af5521234c2cd74ddc4eecd1399e7dc6b7fc9517 100644 (file)
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -19,7 +19,7 @@
 #include <errno.h>
 #include <malloc.h>
 #include <string.h>
-
+#include <stdint.h>
 #include <search.h>
 
 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986
@@ -46,15 +46,12 @@ static int
 isprime (unsigned int number)
 {
   /* no even number will be passed */
-  unsigned int div = 3;
-
-  while (div * div < number && number % div != 0)
-    div += 2;
-
-  return number % div != 0;
+  for (unsigned int div = 3; div <= number / div; div += 2)
+    if (number % div == 0)
+      return 0;
+  return 1;
 }
 
-
 /* Before using the hash table we must allocate memory for it.
    Test for an existing table are done. We allocate one element
    more as the found prime number says. This is done for more effective
@@ -81,10 +78,19 @@ hcreate_r (nel, htab)
      use will not work.  */
   if (nel < 3)
     nel = 3;
-  /* Change nel to the first prime number not smaller as nel. */
-  nel |= 1;      /* make odd */
-  while (!isprime (nel))
-    nel += 2;
+
+  /* Change nel to the first prime number in the range [nel, UINT_MAX - 2],
+     The '- 2' means 'nel += 2' cannot overflow.  */
+  for (nel |= 1; ; nel += 2)
+    {
+      if (UINT_MAX - 2 < nel)
+       {
+         __set_errno (ENOMEM);
+         return 0;
+       }
+      if (isprime (nel))
+       break;
+    }
 
   htab->size = nel;
   htab->filled = 0;