From: Paul Eggert <eggert@cs.ucla.edu>
Date: Sun, 20 Nov 2022 03:04:36 +0000 (-0800)
Subject: copy: fix possible over allocation for regular files
X-Git-Tag: v9.2~96
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6c343a55745a37a1d4d20f39a427170c37044a65;p=thirdparty%2Fcoreutils.git

copy: fix possible over allocation for regular files

* bootstrap.conf (gnulib_modules): Add count-leading-zeros,
which was already an indirect dependency, since ioblksize.h
now uses it directly.
* src/ioblksize.h: Include count-leading-zeros.h.
(io_blksize): Treat impossible blocksizes as IO_BUFSIZE.
When growing a blocksize to IO_BUFSIZE, keep it a multiple of the
stated blocksize.  Work around the ZFS performance bug.
* NEWS: Mention the bug fix.
Problem reported by Korn Andras at https://bugs.gnu.org/59382
---

diff --git a/NEWS b/NEWS
index f43182757f..78a429274a 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,11 @@ GNU coreutils NEWS                                    -*- outline -*-
   'cp -rx / /mnt' no longer complains "cannot create directory /mnt/".
   [bug introduced in coreutils-9.1]
 
+  cp, mv, and install avoid allocating too much memory, and possibly
+  triggering "memory exhausted" failures, on file systems like ZFS,
+  which can return varied file system I/O block size values for files.
+  [bug introduced in coreutils-6.0]
+
   'mv --backup=simple f d/' no longer mistakenly backs up d/f to f~.
   [bug introduced in coreutils-9.1]
 
diff --git a/bootstrap.conf b/bootstrap.conf
index 2477346731..f04c906242 100644
--- a/bootstrap.conf
+++ b/bootstrap.conf
@@ -59,6 +59,7 @@ gnulib_modules="
   config-h
   configmake
   copy-file-range
+  count-leading-zeros
   crypto/md5
   crypto/sha1
   crypto/sha256
diff --git a/src/ioblksize.h b/src/ioblksize.h
index 7a56c1a514..80d7931ba8 100644
--- a/src/ioblksize.h
+++ b/src/ioblksize.h
@@ -18,6 +18,7 @@
 
 /* sys/stat.h and minmax.h will already have been included by system.h. */
 #include "idx.h"
+#include "count-leading-zeros.h"
 #include "stat-size.h"
 
 
@@ -75,8 +76,33 @@ enum { IO_BUFSIZE = 128 * 1024 };
 static inline idx_t
 io_blksize (struct stat sb)
 {
+  /* Treat impossible blocksizes as if they were IO_BUFSIZE.  */
+  idx_t blocksize = ST_BLKSIZE (sb) <= 0 ? IO_BUFSIZE : ST_BLKSIZE (sb);
+
+  /* Use a blocksize of at least IO_BUFSIZE bytes, keeping it a
+     multiple of the original blocksize.  */
+  blocksize += (IO_BUFSIZE - 1) - (IO_BUFSIZE - 1) % blocksize;
+
+  /* For regular files we can ignore the blocksize if we think we know better.
+     ZFS sometimes understates the blocksize, because it thinks
+     apps stupidly allocate a block that large even for small files.
+     This misinformation can cause coreutils to use wrong-sized blocks.
+     Work around some of the performance bug by substituting the next
+     power of two when the reported blocksize is not a power of two.  */
+  if (S_ISREG (sb.st_mode)
+      && blocksize & (blocksize - 1))
+    {
+      int leading_zeros = count_leading_zeros_ll (blocksize);
+      if (IDX_MAX < ULLONG_MAX || leading_zeros)
+        {
+          unsigned long long power = 1ull << (ULLONG_WIDTH - leading_zeros);
+          if (power <= IDX_MAX)
+            blocksize = power;
+        }
+    }
+
   /* Don’t go above the largest power of two that fits in idx_t and size_t,
      as that is asking for trouble.  */
   return MIN (MIN (IDX_MAX, SIZE_MAX) / 2 + 1,
-              MAX (IO_BUFSIZE, ST_BLKSIZE (sb)));
+              blocksize);
 }