xz: Look at resource limits when determining the default memlimit

When no memory usage limits have been set by the user, the default
for multithreaded mode has been 1/4 of total RAM. If this limit is
too high and memory allocation fails, liblzma (and xz) fail. Perhaps
liblzma should handle it better by reducing the number of threads
and continuing with the amount of memory it can allocate, but currently
that isn't the case.

If resource limits were set to about 1/4 of RAM or lower, then xz
could fail for the above reason. This commit makes xz look at
RLIMIT_DATA, RLIMIT_AS, and RLIMIT_VMEM when they are available,
and set the limit 64 MiB below the lowest of those limits. This is
more or less a hack just like the 1/4-of-RAM method is, but this is
simple and quick to implement.

On Linux, there are other limits like cgroup v2 memory.max which
can still make xz fail. The same is likely possible with FreeBSD's
rctl(8).

Co-authored-by: Lasse Collin <lasse.collin@tukaani.org>
Thanks-to: Fangrui Song
Fixes: https://github.com/tukaani-project/xz/issues/195
Closes: https://github.com/tukaani-project/xz/pull/196

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e1499f7e4d70131dac57f1c1a506376bfd55744..f3416e0aa7bcb40c93da2c08b3a77623b21decc4 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2198,6 +2198,9 @@ this many MiB of RAM if xz cannot determine the amount at runtime")
         tuklib_add_definition_if(xz HAVE_OPTRESET)
     endif()
 
+    check_symbol_exists(getrlimit sys/resource.h HAVE_GETRLIMIT)
+    tuklib_add_definition_if(xz HAVE_GETRLIMIT)
+
     check_symbol_exists(posix_fadvise fcntl.h HAVE_POSIX_FADVISE)
     tuklib_add_definition_if(xz HAVE_POSIX_FADVISE)
 

diff --git a/configure.ac b/configure.ac
index 47e2dc65bcdb7b133ae39be6255842b3731e0c3f..3b29cd8477d07209483f845804565b3d7ab9ca05 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1008,8 +1008,8 @@ AC_CHECK_DECL([CLOCK_MONOTONIC], [AC_DEFINE([HAVE_CLOCK_MONOTONIC], [1],
 # Find the best function to set timestamps.
 AC_CHECK_FUNCS([futimens futimes futimesat utimes _futime utime], [break])
 
-# This is nice to have but not mandatory.
-AC_CHECK_FUNCS([posix_fadvise])
+# These are nice to have but not mandatory.
+AC_CHECK_FUNCS([getrlimit posix_fadvise])
 
 TUKLIB_PROGNAME
 TUKLIB_INTEGER

diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index 952652fecb8d994d79d6bc7d29d6257bd262eb5e..2e921474c9c515aed7771ed6196025848bddc6f3 100644 (file)
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -11,6 +11,10 @@
 
 #include "private.h"
 
+#ifdef HAVE_GETRLIMIT
+#      include <sys/resource.h>
+#endif
+
 
 /// Maximum number of worker threads. This can be set with
 /// the --threads=NUM command line option.
@@ -321,6 +325,61 @@ hardware_init(void)
        // /proc/meminfo as the starting point.
        memlimit_mt_default = total_ram / 4;
 
+#ifdef HAVE_GETRLIMIT
+       // Try to set the default multithreaded memory usage limit so that
+       // we won't exceed resource limits. Exceeding the limits would result
+       // in allocation failures, which currently make liblzma and xz fail
+       // (instead of continuing by reducing the number of threads).
+       const int resources[] = {
+               RLIMIT_DATA,
+#      ifdef RLIMIT_AS
+               RLIMIT_AS, // OpenBSD 7.8 doesn't have RLIMIT_AS.
+#      endif
+#      if defined(RLIMIT_VMEM) && RLIMIT_VMEM != RLIMIT_AS
+               RLIMIT_VMEM, // For Solaris. On FreeBSD this is an alias.
+#      endif
+       };
+
+       // The resource limits cannot be passed to liblzma directly;
+       // some margin is required:
+       //   - The memory usage limit counts only liblzma's memory usage,
+       //     but xz itself needs some memory (including gettext usage etc.).
+       //   - Memory allocation has some overhead.
+       //   - Address space limit counts code size too.
+       //
+       // The following value is a guess based on quick testing on Linux.
+       const rlim_t margin = 64 << 20;
+
+       for (size_t i = 0; i < ARRAY_SIZE(resources); ++i) {
+               // RLIM_SAVED_* might be used on some 32-bit OSes
+               // (AIX at least) when the limit doesn't fit in a 32-bit
+               // unsigned integer. Thus, for us these are the same thing
+               // as no limit at all.
+               struct rlimit rl;
+               if (getrlimit(resources[i], &rl) == 0
+                               && rl.rlim_cur != RLIM_INFINITY
+                               && rl.rlim_cur != RLIM_SAVED_CUR
+                               && rl.rlim_cur != RLIM_SAVED_MAX) {
+                       // Subtract the margin from the current resource
+                       // limit, but avoid negative results. Avoid also 0
+                       // because hardware_memlimit_show() (--info-memory)
+                       // treats it specially. In practice, 1 byte is
+                       // effectively 0 anyway.
+                       //
+                       // SUSv2 and POSIX.1-2024 require rlimit_t to be
+                       // unsigned. A cast is needed to silence a compiler
+                       // warning still because, for historical reasons,
+                       // rlim_t is intentionally signed on FreeBSD 14.
+                       const uint64_t rl_with_margin = rl.rlim_cur > margin
+                                       ? (uint64_t)(rl.rlim_cur - margin) : 1;
+
+                       // Lower the memory usage limit if needed.
+                       if (memlimit_mt_default > rl_with_margin)
+                               memlimit_mt_default = rl_with_margin;
+               }
+       }
+#endif
+
 #if SIZE_MAX == UINT32_MAX
        // A too high value may cause 32-bit xz to run out of address space.
        // Use a conservative maximum value here. A few typical address space