From: Paul Eggert <eggert@cs.ucla.edu>
Date: Sun, 4 Aug 2024 05:59:12 +0000 (-0700)
Subject: shuf: avoid integer overflow on huge inputs
X-Git-Tag: v9.6~202
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1ea7255f8b0661cdfabbd13f8f443f81665a07e0;p=thirdparty%2Fcoreutils.git

shuf: avoid integer overflow on huge inputs

* gl/lib/randperm.c: Include <stdckdint.h>.
(randperm_bound): Return SIZE_MAX if the multiplication overflows.
Do not overflow when converting bit count to byte count.
---

diff --git a/gl/lib/randperm.c b/gl/lib/randperm.c
index 50328cd9a5..14a3045241 100644
--- a/gl/lib/randperm.c
+++ b/gl/lib/randperm.c
@@ -23,6 +23,7 @@
 
 #include <limits.h>
 #include <stdbit.h>
+#include <stdckdint.h>
 #include <stdint.h>
 #include <stdlib.h>
 
@@ -39,13 +40,15 @@ randperm_bound (size_t h, size_t n)
 {
   /* Upper bound on number of bits needed to generate the first number
      of the permutation.  */
-  uintmax_t lg_n = stdc_bit_width (n) + 1;
+  unsigned int lg_n = stdc_bit_width (n) + 1;
 
-  /* Upper bound on number of bits needed to generated the first H elements.  */
-  uintmax_t ar = lg_n * h;
+  /* Upper bound on number of bits needed to generate the first H elements.  */
+  uintmax_t ar;
+  if (ckd_mul (&ar, lg_n, h))
+    return SIZE_MAX;
 
   /* Convert the bit count to a byte count.  */
-  size_t bound = (ar + CHAR_BIT - 1) / CHAR_BIT;
+  size_t bound = ar / CHAR_BIT + (ar % CHAR_BIT != 0);
 
   return bound;
 }