regexec.c: avoid arithmetic overflow in buffer size calculation

diff --git a/ChangeLog b/ChangeLog
index c4fb74f09a2cb103ff6f9c7a69334d91023fccfe..9b3fe33f550a76e6b1f5871d8c6c6d3935ec16e2 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 2010-01-22  Jim Meyering  <jim@meyering.net>
 
+       [BZ #11188]
+       * posix/regexec.c (build_trtable): Avoid arithmetic overflow
+       in size calculation.
+
        [BZ #11187]
        * posix/regexec.c (re_search_2_stub): Use simpler method than
        boolean for freeing internal storage.

diff --git a/posix/regexec.c b/posix/regexec.c
index c7d0b37ef50cd6ab50eb8fd38582ae1bb8d9a350..3765d00ffd63c15d8751048ab1c737f345ff4a58 100644 (file)
--- a/posix/regexec.c
+++ b/posix/regexec.c
@@ -3359,6 +3359,13 @@ build_trtable (const re_dfa_t *dfa, re_dfastate_t *state)
   if (BE (err != REG_NOERROR, 0))
     goto out_free;
 
+  /* Avoid arithmetic overflow in size calculation.  */
+  if (BE ((((SIZE_MAX - (sizeof (re_node_set) + sizeof (bitset_t)) * SBC_MAX)
+           / (3 * sizeof (re_dfastate_t *)))
+          < ndests),
+         0))
+    goto out_free;
+
   if (__libc_use_alloca ((sizeof (re_node_set) + sizeof (bitset_t)) * SBC_MAX
                         + ndests * 3 * sizeof (re_dfastate_t *)))
     dest_states = (re_dfastate_t **)