sort: use at most 8 threads by default

* src/sort.c (main): If --parallel isn't specified,
restrict the number of threads to 8 by default.
If the --parallel option is specified, then
allow any number of threads to be set, independent
of the number of processors on the system.
* doc/coreutils.texi (sort invocation): Document the changes
to determining the number of threads to use.
Mention the memory overhead when using multiple threads.
* tests/misc/sort-spinlock-abuse: Allow single core
systems that support pthreads.
* tests/misc/sort-stale-thread-mem: Likewise.
* tests/misc/sort-unique-segv: Likewise.
* NEWS: Mention the change in behaviour.

diff --git a/NEWS b/NEWS
index 484ed5ccca9b5173402e3bcec9836ef14f3b892b..7eda1b2317055a04369a2a150e193ffba1b850f6 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -27,6 +27,12 @@ GNU coreutils NEWS                                    -*- outline -*-
 
   sort -m -o f f ... f no longer dumps core when file descriptors are limited.
 
+** Changes in behavior
+
+  sort will not create more than 8 threads by default due to diminishing
+  performance gains.  Also the --parallel option is no longer restricted
+  to the number of available processors.
+
 ** New features
 
   split accepts the --number option to generate a specific number of files.

diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index 00a55757817dbc78516aaaf4c8f4953780c30b79..a74f64599d331d94a7af3e755d4c46dca1d23526 100644 (file)
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -4190,10 +4190,11 @@ disks and controllers.
 @item --parallel=@var{n}
 @opindex --parallel
 @cindex multithreaded sort
-Limit the number of sorts run in parallel to @var{n}. By default,
-@var{n} is set to the number of available processors, and values
-greater than that are reduced to that limit. Also see
-@ref{nproc invocation}.
+Set the number of sorts run in parallel to @var{n}. By default,
+@var{n} is set to the number of available processors, but limited
+to 8, as there are diminishing performance gains after that.
+Note also that using @var{n} threads increases the memory usage by
+a factor of log @var{n}.  Also see @ref{nproc invocation}.
 
 @item -u
 @itemx --unique

diff --git a/src/sort.c b/src/sort.c
index 54dd8152814a0dab81bc34361d61e8e3bc24a5f8..6cc0588967e0fe0f6854c6cfb8d7a343f90e7ae9 100644 (file)
--- a/src/sort.c
+++ b/src/sort.c
@@ -116,6 +116,10 @@ struct rlimit { size_t rlim_cur; };
    this number has any practical effect.  */
 enum { SUBTHREAD_LINES_HEURISTIC = 4 };
 
+/* The number of threads after which there are
+   diminishing performance gains.  */
+enum { DEFAULT_MAX_THREADS = 8 };
+
 /* Exit statuses.  */
 enum
   {
@@ -455,7 +459,7 @@ Other options:\n\
   -t, --field-separator=SEP  use SEP instead of non-blank to blank transition\n\
   -T, --temporary-directory=DIR  use DIR for temporaries, not $TMPDIR or %s;\n\
                               multiple options specify multiple directories\n\
-      --parallel=N          limit the number of sorts run concurrently to N\n\
+      --parallel=N          change the number sorts run concurrently to N\n\
   -u, --unique              with -c, check for strict ordering;\n\
                               without -c, output only the first of an equal run\n\
 "), DEFAULT_TMPDIR);
@@ -4595,14 +4599,15 @@ main (int argc, char **argv)
     }
   else
     {
-      unsigned long int np2 = num_processors (NPROC_CURRENT_OVERRIDABLE);
-      if (!nthreads || nthreads > np2)
-        nthreads = np2;
+      if (!nthreads)
+        {
+          nthreads = MIN (DEFAULT_MAX_THREADS,
+                          num_processors (NPROC_CURRENT_OVERRIDABLE));
+        }
 
       /* Avoid integer overflow later.  */
       size_t nthreads_max = SIZE_MAX / (2 * sizeof (struct merge_node));
-      if (nthreads_max < nthreads)
-        nthreads = nthreads_max;
+      nthreads = MIN (nthreads, nthreads_max);
 
       sort (files, nfiles, outfile, nthreads);
     }

diff --git a/tests/misc/sort-spinlock-abuse b/tests/misc/sort-spinlock-abuse
index bbf57534bbeed32b4514f68b9db244d9b927a209..67ea895829549d17e1214969750a5d47397f3e1a 100755 (executable)
--- a/tests/misc/sort-spinlock-abuse
+++ b/tests/misc/sort-spinlock-abuse
@@ -20,7 +20,8 @@
 . "${srcdir=.}/init.sh"; path_prepend_ ../src
 print_ver_ sort
 
-test "$(nproc)" = 1 && skip_ "requires a multi-core system"
+grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
+  skip_test_ 'requires pthreads'
 
 seq 100000 > in || framework_failure_
 mkfifo fifo || framework_failure_

diff --git a/tests/misc/sort-stale-thread-mem b/tests/misc/sort-stale-thread-mem
index 2955e22c52316ec08af356bc118bc0cb23ed548f..1f408d4c7faa489e4e0e38cf349ab4b19ab3ede4 100755 (executable)
--- a/tests/misc/sort-stale-thread-mem
+++ b/tests/misc/sort-stale-thread-mem
@@ -24,7 +24,8 @@ print_ver_ sort
 very_expensive_
 
 valgrind --help >/dev/null || skip_ "requires valgrind"
-test "$(nproc)" = 1 && skip_ "requires a multi-core system"
+grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
+  skip_test_ 'requires pthreads'
 
 # gensort output seems to trigger the failure more often,
 # so prefer gensort if it is available.

diff --git a/tests/misc/sort-unique-segv b/tests/misc/sort-unique-segv
index 55a74147fe881856009af5c49ed76bec39bb79de..c4854f9033bc54bfef1cb96d342449f5c46bf9a2 100755 (executable)
--- a/tests/misc/sort-unique-segv
+++ b/tests/misc/sort-unique-segv
@@ -19,7 +19,8 @@
 . "${srcdir=.}/init.sh"; path_prepend_ ../src
 print_ver_ sort
 
-test "$(nproc)" = 1 && skip_ "requires a multi-core system"
+grep '^#define HAVE_PTHREAD_T 1' "$CONFIG_HEADER" > /dev/null ||
+  skip_test_ 'requires pthreads'
 
 cat <<\EOF > in || framework_failure_