#include <config.h>
#include <stdio.h>
#include <sys/types.h>
+#include <sys/uio.h>
#include "system.h"
+#include "alignalloc.h"
#include "full-write.h"
+#include "isapipe.h"
#include "long-options.h"
/* The official name of this program (e.g., no 'g' prefix). */
exit (status);
}
+/* Fill DEST[0..BUFSIZE-1] with repeated copies of SRC[0..SRCSIZE-1],
+ doubling the copy size each iteration. DEST may equal SRC. */
+
+static void
+repeat_pattern (char *dest, char const *src, idx_t srcsize, idx_t bufsize)
+{
+ if (dest != src)
+ memcpy (dest, src, srcsize);
+ for (idx_t filled = srcsize; filled < bufsize; )
+ {
+ idx_t chunk = MIN (filled, bufsize - filled);
+ memcpy (dest + filled, dest, chunk);
+ filled += chunk;
+ }
+}
+
+#if HAVE_SPLICE
+
+/* Empirically determined pipe size for best throughput.
+ Needs to be <= /proc/sys/fs/pipe-max-size */
+enum { SPLICE_PIPE_SIZE = 512 * 1024 };
+
+/* Enlarge a pipe towards SPLICE_PIPE_SIZE and return the actual
+ capacity as a quarter of the pipe size (the empirical sweet spot
+ for vmsplice throughput), rounded down to a multiple of COPYSIZE.
+ Return 0 if the result would be smaller than COPYSIZE. */
+
+static idx_t
+pipe_splice_size (int fd, idx_t copysize)
+{
+ int pipe_cap = 0;
+# if defined F_SETPIPE_SZ && defined F_GETPIPE_SZ
+ if ((pipe_cap = fcntl (fd, F_SETPIPE_SZ, SPLICE_PIPE_SIZE)) < 0)
+ pipe_cap = fcntl (fd, F_GETPIPE_SZ);
+# endif
+ if (pipe_cap <= 0)
+ pipe_cap = 64 * 1024;
+
+ size_t buf_cap = pipe_cap / 4;
+ return buf_cap / copysize * copysize;
+}
+
+#endif
+
+/* Repeatedly write the COPYSIZE-byte pattern in BUF to standard output
+ using vmsplice/splice zero-copy I/O. Since the data never varies,
+ SPLICE_F_GIFT tells the kernel the pages will not be modified.
+
+ Return TRUE if splice I/O was used (caller should check errno and
+ report any error). Return FALSE if splice could not be used. */
+
+static bool
+splice_write (MAYBE_UNUSED char const *buf, MAYBE_UNUSED idx_t copysize)
+{
+ bool output_started = false;
+#if HAVE_SPLICE
+ idx_t page_size = getpagesize ();
+
+ bool stdout_is_pipe = isapipe (STDOUT_FILENO) > 0;
+
+ /* Determine buffer size: enlarge the target pipe,
+ then use 1/4 of actual capacity as the transfer size. */
+ int pipefd[2] = { -1, -1 };
+ idx_t splice_bufsize;
+ char *splice_buf = NULL;
+
+ if (stdout_is_pipe)
+ splice_bufsize = pipe_splice_size (STDOUT_FILENO, copysize);
+ else
+ {
+ if (pipe2 (pipefd, 0) < 0)
+ return false;
+ splice_bufsize = pipe_splice_size (pipefd[0], copysize);
+ }
+
+ if (splice_bufsize == 0)
+ goto done;
+
+ /* Allocate page-aligned buffer for vmsplice.
+ Needed with SPLICE_F_GIFT, but generally good for performance. */
+ if (! (splice_buf = alignalloc (page_size, splice_bufsize)))
+ goto done;
+
+ repeat_pattern (splice_buf, buf, copysize, splice_bufsize);
+
+ /* For the pipe case, vmsplice directly to stdout.
+ For the non-pipe case, vmsplice into the intermediate pipe
+ and then splice from it to stdout. */
+ int vmsplice_fd = stdout_is_pipe ? STDOUT_FILENO : pipefd[1];
+
+ for (;;)
+ {
+ struct iovec iov = { .iov_base = splice_buf,
+ .iov_len = splice_bufsize };
+
+ while (iov.iov_len > 0)
+ {
+ /* Use SPLICE_F_{GIFT,MOVE} to allow the kernel to take references
+ to the pages. I.e., we're indicating we won't make changes.
+ SPLICE_F_GIFT is only appropriate for full pages. */
+ unsigned int flags = iov.iov_len % page_size ? 0 : SPLICE_F_GIFT;
+ ssize_t n = vmsplice (vmsplice_fd, &iov, 1, flags);
+ if (n <= 0)
+ goto done;
+ if (stdout_is_pipe)
+ output_started = true;
+ iov.iov_base = (char *) iov.iov_base + n;
+ iov.iov_len -= n;
+ }
+
+ /* For non-pipe stdout, drain intermediate pipe to stdout. */
+ if (! stdout_is_pipe)
+ {
+ idx_t remaining = splice_bufsize;
+ while (remaining > 0)
+ {
+ ssize_t s = splice (pipefd[0], NULL, STDOUT_FILENO, NULL,
+ remaining, SPLICE_F_MOVE);
+ if (s <= 0)
+ goto done;
+ output_started = true;
+ remaining -= s;
+ }
+ }
+ }
+
+done:
+ if (pipefd[0] >= 0)
+ {
+ int saved_errno = errno;
+ close (pipefd[0]);
+ close (pipefd[1]);
+ errno = saved_errno;
+ }
+ alignfree (splice_buf);
+#endif
+ return output_started;
+}
+
int
main (int argc, char **argv)
{
while (++operandp < operand_lim);
buf[bufused - 1] = '\n';
- /* If a larger buffer was allocated, fill it by repeating the buffer
- contents. */
- size_t copysize = bufused;
- for (size_t copies = bufalloc / copysize; --copies; )
+ idx_t copysize = bufused;
+
+ /* Repeatedly output the buffer until there is a write error; then fail.
+ Do a minimal write first to check output with minimal set up cost.
+ If successful then set up for efficient repetition. */
+ if (full_write (STDOUT_FILENO, buf, copysize) == copysize
+ && splice_write (buf, copysize) == 0)
{
- memcpy (buf + bufused, buf, copysize);
- bufused += copysize;
+ /* If a larger buffer was allocated, fill it by repeated copies. */
+ bufused = bufalloc / copysize * copysize;
+ if (bufused > copysize)
+ repeat_pattern (buf, buf, copysize, bufused);
+ while (full_write (STDOUT_FILENO, buf, bufused) == bufused)
+ continue;
}
- /* Repeatedly output the buffer until there is a write error; then fail. */
- while (full_write (STDOUT_FILENO, buf, bufused) == bufused)
- continue;
error (0, errno, _("standard output"));
main_exit (EXIT_FAILURE);
}
projects / thirdparty / coreutils.git / commitdiff
