Handle overflow in __hcreate_r

Hi,

As in bugzilla entry there is overflow in hsearch when looking for prime
number as SIZE_MAX - 1 is divisible by 5. We fix that by rejecting large
inputs before looking for prime.

	* misc/hsearch_r.c (__hcreate_r): Handle overflow.

(cherry picked from commit 2f5c1750558fe64bac361f52d6827ab1bcfe52bc)

diff --git a/ChangeLog b/ChangeLog
index 9740c89cab786e99834cff0e75bee7c5dce599ff..e8189959698adee6df417a42065dc272548cf3bb 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2015-08-25  Ondřej Bílka  <neleai@seznam.cz>
+
+       [BZ #18240]
+       * misc/hsearch_r.c (__hcreate_r): Handle overflow.
+
 2015-10-27  Ludovic Courtès  <ludo@gnu.org>
 
        * locale/loadlocale.c (_nl_intern_locale_data): Change assertion

diff --git a/misc/hsearch_r.c b/misc/hsearch_r.c
index 9f55e845cf7800e370b3d476655a1e5a86f0055d..559df29cf7b124dbdf88f089f1ed5a784b364105 100644 (file)
--- a/misc/hsearch_r.c
+++ b/misc/hsearch_r.c
@@ -19,7 +19,7 @@
 #include <errno.h>
 #include <malloc.h>
 #include <string.h>
-
+#include <stdint.h>
 #include <search.h>
 
 /* [Aho,Sethi,Ullman] Compilers: Principles, Techniques and Tools, 1986
@@ -73,6 +73,13 @@ __hcreate_r (nel, htab)
       return 0;
     }
 
+  if (nel >= SIZE_MAX / sizeof (_ENTRY))
+    {
+      __set_errno (ENOMEM);
+      return 0;
+    }
+
+
   /* There is still another table active. Return with error. */
   if (htab->table != NULL)
     return 0;