From: Janne Blomqvist <jb@gcc.gnu.org>
Date: Wed, 3 Jan 2018 12:08:05 +0000 (+0200)
Subject: PR libgfortran/83649 Chunk large reads and writes
X-Git-Tag: releases/gcc-6.5.0~611
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7408aebbccb3321f47b559d606ef0ec644f298d9;p=thirdparty%2Fgcc.git

PR libgfortran/83649 Chunk large reads and writes

Backport from trunk.

It turns out that Linux never reads or writes more than 2147479552
bytes in a single syscall. For writes this is not a problem as
libgfortran already contains a loop around write() to handle short
writes. But for reads we cannot do this, since then read will hang if
we have a short read when reading from the terminal.  Also, there are
reports that macOS fails I/O's larger than 2 GB.  Thus, to work around
these issues do large reads/writes in chunks.

The testcase from the PR

program largewr
  integer(kind=1) :: a(2_8**31+1)
  a = 0
  a(size(a, kind=8)) = 1
  open(10, file="largewr.dat", access="stream", form="unformatted")
  write (10) a
  close(10)
  a(size(a, kind=8)) = 2
  open(10, file="largewr.dat", access="stream", form="unformatted")
  read (10) a
  if (a(size(a, kind=8)) == 1) then
    print *, "All is well"
  else
    print *, "Oh no"
  end if
end program largewr

fails on trunk but works with the patch.

Regtested on x86_64-pc-linux-gnu, committed to trunk.

libgfortran/ChangeLog:

2018-01-03  Janne Blomqvist  <jb@gcc.gnu.org>

	PR libgfortran/83649
	* io/unix.c (MAX_CHUNK): New define.
	(raw_read): For reads larger than MAX_CHUNK, loop.
	(raw_write): Write no more than MAX_CHUNK bytes per iteration.

From-SVN: r256173
---

diff --git a/libgfortran/ChangeLog b/libgfortran/ChangeLog
index 2e73dd1002d9..2bb438304780 100644
--- a/libgfortran/ChangeLog
+++ b/libgfortran/ChangeLog
@@ -1,3 +1,11 @@
+2018-01-03  Janne Blomqvist  <jb@gcc.gnu.org>
+
+	Backport from trunk
+	PR libgfortran/83649
+	* io/unix.c (MAX_CHUNK): New define.
+	(raw_read): For reads larger than MAX_CHUNK, loop.
+	(raw_write): Write no more than MAX_CHUNK bytes per iteration.
+
 2017-12-16  Jerry DeLisle  <jvdelisle@gcc.gnu.org>
 
 	Backport from trunk
diff --git a/libgfortran/io/unix.c b/libgfortran/io/unix.c
index bdec1e89f52e..aed668471a82 100644
--- a/libgfortran/io/unix.c
+++ b/libgfortran/io/unix.c
@@ -294,12 +294,50 @@ raw_flush (unix_stream * s  __attribute__ ((unused)))
   return 0;
 }
 
+/* Write/read at most 2 GB - 4k chunks at a time. Linux never reads or
+   writes more than this, and there are reports that macOS fails for
+   larger than 2 GB as well.  */
+#define MAX_CHUNK 2147479552
+
 static ssize_t
 raw_read (unix_stream * s, void * buf, ssize_t nbyte)
 {
   /* For read we can't do I/O in a loop like raw_write does, because
-     that will break applications that wait for interactive I/O.  */
-  return read (s->fd, buf, nbyte);
+     that will break applications that wait for interactive I/O.  We
+     still can loop around EINTR, though.  This however causes a
+     problem for large reads which must be chunked, see comment above.
+     So assume that if the size is larger than the chunk size, we're
+     reading from a file and not the terminal.  */
+  if (nbyte <= MAX_CHUNK)
+    {
+      while (true)
+	{
+	  ssize_t trans = read (s->fd, buf, nbyte);
+	  if (trans == -1 && errno == EINTR)
+	    continue;
+	  return trans;
+	}
+    }
+  else
+    {
+      ssize_t bytes_left = nbyte;
+      char *buf_st = buf;
+      while (bytes_left > 0)
+	{
+	  ssize_t to_read = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+	  ssize_t trans = read (s->fd, buf_st, to_read);
+	  if (trans == -1)
+	    {
+	      if (errno == EINTR)
+		continue;
+	      else
+		return trans;
+	    }
+	  buf_st += trans;
+	  bytes_left -= trans;
+	}
+      return nbyte - bytes_left;
+    }
 }
 
 static ssize_t
@@ -312,10 +350,13 @@ raw_write (unix_stream * s, const void * buf, ssize_t nbyte)
   buf_st = (char *) buf;
 
   /* We must write in a loop since some systems don't restart system
-     calls in case of a signal.  */
+     calls in case of a signal.  Also some systems might fail outright
+     if we try to write more than 2 GB in a single syscall, so chunk
+     up large writes.  */
   while (bytes_left > 0)
     {
-      trans = write (s->fd, buf_st, bytes_left);
+      ssize_t to_write = bytes_left < MAX_CHUNK ? bytes_left: MAX_CHUNK;
+      trans = write (s->fd, buf_st, to_write);
       if (trans < 0)
 	{
 	  if (errno == EINTR)