xmalloca, etc.: avoid unlikely trap

* lib/xmalloca.h (xnmalloca):
Use size_t, not ptrdiff_t, to avoid implementation-defined
behavior (which could include a trap) if N is 0 and S exceeds
PTRDIFF_MAX.
* lib/xalloc-oversized.h: Adjust comment to match.

diff --git a/ChangeLog b/ChangeLog
index 63471166c4ba46f9a55b9ddf183c81fb75908d31..f91f2e3df697d0a81c6c996e33417e496f523e6f 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,13 @@
 2021-04-24  Paul Eggert  <eggert@cs.ucla.edu>
 
-       xmalloca, etc.: fix some xalloc-oversized issues
+       xmalloca, etc.: avoid unlikely trap
        * lib/malloca.h (nmalloca):
+       * lib/xmalloca.h (xnmalloca):
+       Use size_t, not ptrdiff_t, to avoid implementation-defined
+       behavior (which could include a trap) if N is 0 and S exceeds
+       PTRDIFF_MAX.
+       * lib/xalloc-oversized.h: Adjust comment to match.
+
        * lib/xmalloca.h (xnmalloca): Convert S to ptrdiff_t to avoid
        arithmetic overflow if N and S are both narrower than ptrdiff_t.
        * lib/xalloc-oversized.h (xalloc_oversized):

diff --git a/lib/xalloc-oversized.h b/lib/xalloc-oversized.h
index 62f1ae94b8fff4cbecfd918318027909b969d5fd..2619a2aab40d7c50ab5501779e60d19dde8ca822 100644 (file)
--- a/lib/xalloc-oversized.h
+++ b/lib/xalloc-oversized.h
@@ -38,8 +38,8 @@
 
    Warning: (xalloc_oversized (N, S) ? NULL : malloc (N * S)) can
    misbehave if N and S are both narrower than ptrdiff_t and size_t,
-   and can be rewritten as (xalloc_oversized (N, S) ?  NULL : malloc
-   (N * (ptrdiff_t) S)) or similarly with size_t.
+   and can be rewritten as (xalloc_oversized (N, S) ?  NULL
+   : malloc (N * (size_t) S)).
 
    This is a macro, not a function, so that it works even if an
    argument exceeds MAX (PTRDIFF_MAX, SIZE_MAX).  */

diff --git a/lib/xmalloca.h b/lib/xmalloca.h
index c7815f61717030848aa6d707286f3251285968c5..dbc648697e95d8507bdf64f5384fdb33a07f323e 100644 (file)
--- a/lib/xmalloca.h
+++ b/lib/xmalloca.h
@@ -54,7 +54,7 @@ extern void * xmmalloca (size_t n);
 #if HAVE_ALLOCA
 /* Rely on xmalloca (SIZE_MAX) calling xalloc_die ().  */
 # define xnmalloca(n, s) \
-    xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (ptrdiff_t) (s))
+    xmalloca (xalloc_oversized (n, s) ? (size_t) (-1) : (n) * (size_t) (s))
 #else
 # define xnmalloca(n, s) \
     xnmalloc (n, s)