du: read and process --files0-from= input a name at a time,

rather than by reading the entire input into memory and *then*
processing each file name.
* src/du.c: Include "argv-iter.h", not "readtokens0.h".
(main): Rewrite to use argv-iter.
Call xfts_open on each argument, rather than on the entire
argv list at once.
Call print_size here, not from du_files.
Diagnose read failure.
* NEWS (Bug fixes): Mention it.
* THANKS: update.
Reported by Barry Kelly.  More details in
http://article.gmane.org/gmane.comp.gnu.core-utils.bugs/15159/

diff --git a/NEWS b/NEWS
index f0c55500f9186881c23e1d10bb30e263e08d99fc..158b089ad97c3dfa04c9967dae3a9e301c5791f7 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -12,6 +12,9 @@ GNU coreutils NEWS                                    -*- outline -*-
 
   cp uses much less memory in some situations
 
+  du --files0-from=FILE no longer reads all of FILE into RAM before
+  processing the first file name
+
   seq 9223372036854775807 9223372036854775808 now prints only two numbers
   on systems with extended long double support and good library support.
   Even with this patch, on some systems, it still produces invalid output,

diff --git a/THANKS b/THANKS
index d06f755a0dadf6cd886baf97179f5a68cd66ddb0..f74d4fb48508518fa5083db87893f1fecd1a40f0 100644 (file)
--- a/THANKS
+++ b/THANKS
@@ -58,6 +58,7 @@ Augey Mikus                         mikus@dqc.org
 Aurelien Jarno                      aurel32@debian.org
 Austin Donnelly                     Austin.Donnelly@cl.cam.ac.uk
 Axel Kittenberger                   Anshil@gmx.net
+Barry Kelly                         http://barrkel.blogspot.com/
 Bauke Jan Douma                     bjdouma@xs4all.nl
 Ben Elliston                        bje@air.net.au
 Ben Harris                          bjh21@netbsd.org

diff --git a/src/du.c b/src/du.c
index e5669786de504b37c29788ac0fa86c452eddbf40..6e4d28b64fa495f947a2265cd9470ac9338b3392 100644 (file)
--- a/src/du.c
+++ b/src/du.c
@@ -30,6 +30,7 @@
 #include <assert.h>
 #include "system.h"
 #include "argmatch.h"
+#include "argv-iter.h"
 #include "error.h"
 #include "exclude.h"
 #include "fprintftime.h"
@@ -37,7 +38,6 @@
 #include "human.h"
 #include "quote.h"
 #include "quotearg.h"
-#include "readtokens0.h"
 #include "same.h"
 #include "stat-time.h"
 #include "xfts.h"
@@ -649,9 +649,6 @@ du_files (char **files, int bit_flags)
       fts_close (fts);
     }
 
-  if (print_grand_total)
-    print_size (&tot_dui, _("total"));
-
   return ok;
 }
 
@@ -660,10 +657,8 @@ main (int argc, char **argv)
 {
   char *cwd_only[2];
   bool max_depth_specified = false;
-  char **files;
   bool ok = true;
   char *files_from = NULL;
-  struct Tokens tok;
 
   /* Bit flags that control how fts works.  */
   int bit_flags = FTS_TIGHT_CYCLE_CHECK;
@@ -926,6 +921,7 @@ main (int argc, char **argv)
         }
     }
 
+  struct argv_iterator *ai;
   if (files_from)
     {
       /* When using --files0-from=F, you may not specify any files
@@ -942,78 +938,96 @@ main (int argc, char **argv)
        error (EXIT_FAILURE, errno, _("cannot open %s for reading"),
               quote (files_from));
 
-      readtokens0_init (&tok);
-
-      if (! readtokens0 (stdin, &tok) || fclose (stdin) != 0)
-       error (EXIT_FAILURE, 0, _("cannot read file names from %s"),
-              quote (files_from));
-
-      files = tok.tok;
+      ai = argv_iter_init_stream (stdin);
     }
   else
     {
-      files = (optind < argc ? argv + optind : cwd_only);
+      char **files = (optind < argc ? argv + optind : cwd_only);
+      ai = argv_iter_init_argv (files);
     }
 
+  if (!ai)
+    xalloc_die ();
+
   /* Initialize the hash structure for inode numbers.  */
   hash_init ();
 
-  /* Report and filter out any empty file names before invoking fts.
-     This works around a glitch in fts, which fails immediately
-     (without looking at the other file names) when given an empty
-     file name.  */
-  {
-    size_t i = 0;
-    size_t j;
+  bit_flags |= symlink_deref_bits;
+  static char *temp_argv[] = { NULL, NULL };
 
-    for (j = 0; ; j++)
-      {
-       if (i != j)
-         files[i] = files[j];
+  while (true)
+    {
+      bool skip_file = false;
+      enum argv_iter_err ai_err;
+      char *file_name = argv_iter (ai, &ai_err);
+      if (ai_err == AI_ERR_EOF)
+       break;
+      if (!file_name)
+       {
+         switch (ai_err)
+           {
+           case AI_ERR_READ:
+             error (0, errno, _("%s: read error"), quote (files_from));
+             skip_file = true;
+             continue;
 
-       if ( ! files[i])
-         break;
+           case AI_ERR_MEM:
+             xalloc_die ();
 
-       if (files_from && STREQ (files_from, "-") && STREQ (files[i], "-"))
-         {
-           /* Give a better diagnostic in an unusual case:
-              printf - | du --files0-from=- */
-           error (0, 0, _("when reading file names from stdin, "
-                          "no file name of %s allowed"),
-                  quote ("-"));
-           continue;
-         }
+           default:
+             assert (!"unexpected error code from argv_iter");
+           }
+       }
+      if (files_from && STREQ (files_from, "-") && STREQ (file_name, "-"))
+       {
+         /* Give a better diagnostic in an unusual case:
+            printf - | du --files0-from=- */
+         error (0, 0, _("when reading file names from stdin, "
+                        "no file name of %s allowed"),
+                quote (file_name));
+         skip_file = true;
+       }
 
-       if (files[i][0])
-         i++;
-       else
-         {
-           /* Diagnose a zero-length file name.  When it's one
-              among many, knowing the record number may help.  */
-           if (files_from)
-             {
-               /* Using the standard `filename:line-number:' prefix here is
-                  not totally appropriate, since NUL is the separator, not NL,
-                  but it might be better than nothing.  */
-               unsigned long int file_number = j + 1;
-               error (0, 0, "%s:%lu: %s", quotearg_colon (files_from),
-                      file_number, _("invalid zero-length file name"));
-             }
-           else
-             error (0, 0, "%s", _("invalid zero-length file name"));
-         }
-      }
+      /* Report and skip any empty file names before invoking fts.
+        This works around a glitch in fts, which fails immediately
+        (without looking at the other file names) when given an empty
+        file name.  */
+      if (!file_name[0])
+       {
+         /* Diagnose a zero-length file name.  When it's one
+            among many, knowing the record number may help.
+            FIXME: currently print the record number only with
+            --files0-from=FILE.  Maybe do it for argv, too?  */
+         if (files_from == NULL)
+           error (0, 0, "%s", _("invalid zero-length file name"));
+         else
+           {
+             /* Using the standard `filename:line-number:' prefix here is
+                not totally appropriate, since NUL is the separator, not NL,
+                but it might be better than nothing.  */
+             unsigned long int file_number = argv_iter_n_args (ai);
+             error (0, 0, "%s:%lu: %s", quotearg_colon (files_from),
+                    file_number, _("invalid zero-length file name"));
+           }
+         skip_file = true;
+       }
+
+      if (skip_file)
+       ok = false;
+      else
+       {
+         temp_argv[0] = file_name;
+         ok &= du_files (temp_argv, bit_flags);
+       }
+    }
 
-    ok = (i == j);
-  }
+  argv_iter_free (ai);
 
-  bit_flags |= symlink_deref_bits;
-  ok &= du_files (files, bit_flags);
+  if (files_from && (ferror (stdin) || fclose (stdin) != 0))
+    error (EXIT_FAILURE, 0, _("error reading %s"), quote (files_from));
 
-  /* This isn't really necessary, but it does ensure we
-     exercise this function.  */
-  if (files_from)
-    readtokens0_free (&tok);
+  if (print_grand_total)
+    print_size (&tot_dui, _("total"));
 
   hash_free (htab);