rm, du, chmod, chown, chgrp: use much less memory for large directories

For details, see the gnulib commit,
http://git.sv.gnu.org/cgit/gnulib.git/commit/?id=47cb657e
* tests/rm/4-million-entry-dir: New test.
* tests/Makefile.am (TESTS): Add it.
* NEWS (Bug fixes): Mention it.
* gnulib: Update to latest to get the required fts fixes.

diff --git a/NEWS b/NEWS
index 6e24f5cb1a5462f7fde2e28d0b8691d4aa58c361..b356a039c3ff152b47626b921ac157613e9cc5b3 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -17,6 +17,15 @@ GNU coreutils NEWS                                    -*- outline -*-
   to dst/s/b rather than simply linking dst/s/b to dst/s/a.
   [This bug appears to have been present in "the beginning".]
 
+  fts-using tools (rm, du, chmod, chgrp, chown, chcon) no longer use memory
+  proportional to the number of entries in each directory they process.
+  Before, rm -rf 4-million-entry-directory would consume about 1GiB of memory.
+  Now, it uses less than 30GB, no matter how many entries there are.
+  [this bug was inherent in the use of fts: thus, for rm the bug was
+  introduced in coreutils-8.0.  The prior implementation of rm did not use
+  as much memory.  du, chmod, chgrp and chown started using fts in 6.0.
+  chcon was added in coreutils-6.9.91 with fts support.  ]
+
   printf '%d' '"' no longer accesses out-of-bounds memory in the diagnostic.
   [bug introduced in sh-utils-1.16]
 

diff --git a/gnulib b/gnulib
index d2b8ab669f3129ac0d349eead1217adc38d795eb..47cb657eca1abf2c26c32c8ce03def994a3ee37c 160000 (submodule)
--- a/gnulib
+++ b/gnulib
@@ -1 +1 @@
-Subproject commit d2b8ab669f3129ac0d349eead1217adc38d795eb
+Subproject commit 47cb657eca1abf2c26c32c8ce03def994a3ee37c

diff --git a/tests/Makefile.am b/tests/Makefile.am
index eb67512c95e78cb3cbbec0a16dcf484c89b5162a..f0200e1221a9585a03dd39cf5f08e174e0f19600 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -135,6 +135,7 @@ TESTS =                                             \
   rm/unread3                                   \
   rm/unreadable                                        \
   rm/v-slash                                   \
+  rm/4-million-entry-dir                       \
   chgrp/default-no-deref                       \
   chgrp/deref                                  \
   chgrp/no-x                                   \

diff --git a/tests/rm/4-million-entry-dir b/tests/rm/4-million-entry-dir
new file mode 100755 (executable)
index 0000000..23130a6
--- /dev/null
+++ b/tests/rm/4-million-entry-dir
@@ -0,0 +1,35 @@
+#!/bin/sh
+# in coreutils-8.12, this would have required ~1GB of memory
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+print_ver_ rm
+
+very_expensive_
+
+# Put 4M files in a directory.
+mkdir d && cd d || framework_failure_
+seq 4000000|xargs touch || framework_failure_
+
+cd ..
+
+# Restricted to 50MB, rm from coreutils-8.12 would fail with a
+# diagnostic like "rm: fts_read failed: Cannot allocate memory".
+ulimit -v 50000
+rm -rf d || fail=1
+
+Exit $fail