fix: Fix reading of files larger than 2^31 bytes

For some kernels read(2) makes a short read for counts larger than 2^31,
even for files on 64-bit architectures. Other kernels seem to not like
large counts at all, returning a failure.

Fix this by limiting the read buffer to a smaller value. This also
removes a short read optimization which should have been removed in
cff3bf417420ea88fe53e7267ae2a356898ae326.

diff --git a/src/util/file.cpp b/src/util/file.cpp
index 51a99984431e69c120a0de6a30d22a0f9459b13f..5b776297fec3262d6962683a4be0da38cefc17f8 100644 (file)
--- a/src/util/file.cpp
+++ b/src/util/file.cpp
@@ -240,9 +240,6 @@ read_file(const fs::path& path, size_t size_hint)
     size_hint = de.size();
   }
 
-  // +1 to be able to detect EOF in the first read call
-  size_hint = (size_hint < 1024) ? 1024 : size_hint + 1;
-
   const int open_flags = [] {
     if constexpr (std::is_same<T, std::string>::value) {
       return O_RDONLY | O_TEXT;
@@ -258,22 +255,19 @@ read_file(const fs::path& path, size_t size_hint)
   int64_t ret = 0;
   size_t pos = 0;
   T result;
-  result.resize(size_hint);
+  result.resize(std::max(size_t{1024}, size_hint + 1));
 
   while (true) {
     if (pos == result.size()) {
       result.resize(2 * result.size());
     }
-    const size_t max_read = result.size() - pos;
+    const size_t max_read = std::min(size_t{1024 * 1024}, result.size() - pos);
     ret = read(*fd, &result[pos], max_read);
     if (ret == 0 || (ret == -1 && errno != EINTR)) {
       break;
     }
     if (ret > 0) {
       pos += ret;
-      if (static_cast<size_t>(ret) < max_read) {
-        break;
-      }
     }
   }