From: Florian Krohm <florian@eich-krohm.de>
Date: Sat, 21 Mar 2015 10:58:37 +0000 (+0000)
Subject: Change the GCC demangler to not use VLA. The rationale is that these VLAs
X-Git-Tag: svn/VALGRIND_3_11_0~563
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=23c60d25679af558d0be91323062fc042b70b50a;p=thirdparty%2Fvalgrind.git

Change the GCC demangler to not use VLA. The rationale is that these VLAs
are allocated on the stack and they can become quite large - in particular
when the client is a C++ application using the Boost library.
In combination with the demanglers recursive nature this can quickly lead
to exhaustion of valgrind's per-thread stack (which cannot be dynamically
grown). Additionally, due to the large VLAs (I've seen a 32k array) we
could run out of stack space without issuing a prior warning and instead
just segfault.
Therefore this patch allocates these arrays on the heap and frees them
later. Basically this is a respin of Joseph's r10385.


git-svn-id: svn://svn.valgrind.org/valgrind/trunk@15029
---

diff --git a/coregrind/m_demangle/cp-demangle.c b/coregrind/m_demangle/cp-demangle.c
index e311e3f5e3..381bd4d5f7 100644
--- a/coregrind/m_demangle/cp-demangle.c
+++ b/coregrind/m_demangle/cp-demangle.c
@@ -4070,6 +4070,7 @@ cplus_demangle_print_callback (int options,
   d_print_init (&dpi, callback, opaque, dc);
 
   {
+#if 0 /* in valgrind */
 #ifdef CP_DYNAMIC_ARRAYS
     __extension__ struct d_saved_scope scopes[dpi.num_saved_scopes ?: 1];
     __extension__ struct d_print_template temps[dpi.num_copy_templates ?: 1];
@@ -4082,13 +4083,28 @@ cplus_demangle_print_callback (int options,
     dpi.copy_templates = alloca (dpi.num_copy_templates
 				 * sizeof (*dpi.copy_templates));
 #endif
-
+#else
+    /* Allocate memory dynamically to avoid VLAs as valgrind stack
+       is a scarce resource */
+    dpi.saved_scopes = xmalloc(dpi.num_saved_scopes
+			       * sizeof (*dpi.saved_scopes));
+    dpi.copy_templates = xmalloc (dpi.num_copy_templates
+                                  * sizeof (*dpi.copy_templates));
+#endif /* ! in valgrind */
     d_print_comp (&dpi, options, dc);
   }
 
   d_print_flush (&dpi);
 
-  return ! d_print_saw_error (&dpi);
+  int status = ! d_print_saw_error (&dpi);
+
+#if 0 /* in valgrind */
+#else
+  free (dpi.saved_scopes);
+  free (dpi.copy_templates);
+#endif  /* in valgrind */
+
+  return status;
 }
 
 /* Turn components into a human readable string.  OPTIONS is the
@@ -5863,6 +5879,7 @@ d_demangle_callback (const char *mangled, int options,
   cplus_demangle_init_info (mangled, options, strlen (mangled), &di);
 
   {
+#if 0 /* in valgrind */
 #ifdef CP_DYNAMIC_ARRAYS
     __extension__ struct demangle_component comps[di.num_comps];
     __extension__ struct demangle_component *subs[di.num_subs];
@@ -5873,6 +5890,12 @@ d_demangle_callback (const char *mangled, int options,
     di.comps = alloca (di.num_comps * sizeof (*di.comps));
     di.subs = alloca (di.num_subs * sizeof (*di.subs));
 #endif
+#else
+    /* Allocate memory dynamically to avoid VLAs as valgrind stack
+       is a scarce resource */
+    di.comps = xmalloc (di.num_comps * sizeof (*di.comps));
+    di.subs = xmalloc (di.num_subs * sizeof (*di.subs));
+#endif /* ! in valgrind */
 
     switch (type)
       {
@@ -5913,6 +5936,12 @@ d_demangle_callback (const char *mangled, int options,
              : 0;
   }
 
+#if 0 /* in valgrind */
+#else
+  free (di.comps);
+  free (di.subs);
+#endif  /* in valgrind */
+
   return status;
 }
 
@@ -6144,6 +6173,7 @@ is_ctor_or_dtor (const char *mangled,
   cplus_demangle_init_info (mangled, DMGL_GNU_V3, strlen (mangled), &di);
 
   {
+#if 0 /* in valgrind */
 #ifdef CP_DYNAMIC_ARRAYS
     __extension__ struct demangle_component comps[di.num_comps];
     __extension__ struct demangle_component *subs[di.num_subs];
@@ -6154,7 +6184,12 @@ is_ctor_or_dtor (const char *mangled,
     di.comps = alloca (di.num_comps * sizeof (*di.comps));
     di.subs = alloca (di.num_subs * sizeof (*di.subs));
 #endif
-
+#else
+    /* Allocate memory dynamically to avoid VLAs as valgrind stack
+       is a scarce resource */
+    di.comps = xmalloc (di.num_comps * sizeof (*di.comps));
+    di.subs = xmalloc (di.num_subs * sizeof (*di.subs));
+#endif /* ! in valgrind */
     dc = cplus_demangle_mangled_name (&di, 1);
 
     /* Note that because we did not pass DMGL_PARAMS, we don't expect
@@ -6196,6 +6231,12 @@ is_ctor_or_dtor (const char *mangled,
       }
   }
 
+#if 0 /* in valgrind */
+#else
+  free (di.comps);
+  free (di.subs);
+#endif  /* in valgrind */
+
   return ret;
 }