stdbuf: don’t allocate more than SIZE_MAX / 2

* src/libstdbuf.c (apply_mode): Be more conservative about
sizes passed to malloc, since we can’t rely on Gnulib malloc.

diff --git a/src/libstdbuf.c b/src/libstdbuf.c
index 2b4dbe1f2d47b045260294635fdd80852523cd1c..51665bd5a0fe53d96969e62332e11e834be086eb 100644 (file)
--- a/src/libstdbuf.c
+++ b/src/libstdbuf.c
@@ -94,8 +94,11 @@ apply_mode (FILE *stream, char const *stream_name, char const *envvar)
           return;
         }
 
-      buf = (size <= ((unsigned long int) -2 < (size_t) -1
-                      ? (unsigned long int) -2 : (size_t) -1)
+      /* If strtoul might have overflowed or if the size is more than
+         half of size_t range, treat it as an allocation failure.
+         Huge sizes can cause problems with some stdio implementations.  */
+      buf = (size <= ((unsigned long int) -2 < (size_t) -1 / 2
+                      ? (unsigned long int) -2 : (size_t) -1 / 2)
              ? malloc (size) : nullptr);
       if (!buf)
         {