Improve memory usage limits, esp on 32-bit systems and an easy to use manual memory...

Merge of master-melb:xfs-cmds:30906a by kenmcd.

  Document new -m memory limit option for xfs_repair

diff --git a/man/man8/xfs_repair.8 b/man/man8/xfs_repair.8
index 4bd7051276b507037a3019261090409433b9ebe4..8784c6e1d193c30142852b66a195a020ea8d5887 100644 (file)
--- a/man/man8/xfs_repair.8
+++ b/man/man8/xfs_repair.8
@@ -6,6 +6,9 @@ xfs_repair \- repair an XFS filesystem
 [
 .B \-dfLnPv
 ] [
+.B \-m
+.I maxmem
+] [
 .B \-o
 .I subopt\c
 [\c
@@ -85,6 +88,18 @@ No modify mode. Specifies that
 .B xfs_repair
 should not modify the filesystem but should only scan the
 filesystem and indicate what repairs would have been made.
+.TP
+.BI \-m " maxmem"
+Specifies the approximate maximum amount of memory, in megabytes, to use for
+.BR xfs_repair .
+.B xfs_repair
+has its own internal block cache which will scale out up to the lesser of the
+process's virtual address limit or about 75% of the system's physical RAM.
+This option overrides these limits.
+.IP
+.B NOTE:
+These memory limits are only approximate and may use more than the specified
+limit.
 .HP
 .B \-o
 .I subopt\c

diff --git a/repair/xfs_repair.c b/repair/xfs_repair.c
index dd5b6247b67fdbc91cb24f0284fa2c7843e96f53..0e7e564ea823f52970f84d85e53cd4f69eebe7b0 100644 (file)
--- a/repair/xfs_repair.c
+++ b/repair/xfs_repair.c
@@ -17,6 +17,7 @@
  */
 
 #include <xfs/libxlog.h>
+#include <sys/resource.h>
 #include "avl.h"
 #include "avl64.h"
 #include "globals.h"
@@ -66,12 +67,13 @@ char *o_opts[] = {
 
 static int     ihash_option_used;
 static int     bhash_option_used;
+static long    max_mem_specified;      /* in megabytes */
 
 static void
 usage(void)
 {
        do_warn(
-_("Usage: %s [-nLvV] [-o subopt[=value]] [-l logdev] [-r rtdev] devname\n"),
+_("Usage: %s [-nLvV] [-m memMB] [-o subopt[=value]] [-l logdev] [-r rtdev] devname\n"),
                progname);
        exit(1);
 }
@@ -191,7 +193,7 @@ process_args(int argc, char **argv)
         * XXX have to add suboption processing here
         * attributes, quotas, nlinks, aligned_inos, sb_fbits
         */
-       while ((c = getopt(argc, argv, "o:fl:r:LnDvVdPMt:")) != EOF)  {
+       while ((c = getopt(argc, argv, "o:fl:m:r:LnDvVdPt:")) != EOF)  {
                switch (c) {
                case 'D':
                        dumpcore = 1;
@@ -222,6 +224,9 @@ process_args(int argc, char **argv)
                                        ihash_option_used = 1;
                                        break;
                                case BHASH_SIZE:
+                                       if (max_mem_specified)
+                                               do_abort(
+                       _("-o bhash option cannot be used with -m option\n"));
                                        libxfs_bhash_size = (int) strtol(val, 0, 0);
                                        bhash_option_used = 1;
                                        break;
@@ -245,6 +250,12 @@ process_args(int argc, char **argv)
                case 'f':
                        isa_file = 1;
                        break;
+               case 'm':
+                       if (bhash_option_used)
+                               do_abort(_("-m option cannot be used with "
+                                               "-o bhash option\n"));
+                       max_mem_specified = strtol(optarg, 0, 0);
+                       break;
                case 'L':
                        zap_log = 1;
                        break;
@@ -550,9 +561,10 @@ main(int argc, char **argv)
         * Calculations are done in kilobyte units.
         */
 
-       if (!bhash_option_used) {
+       if (!bhash_option_used || max_mem_specified) {
                unsigned long   mem_used;
-               unsigned long   phys_mem;
+               unsigned long   max_mem;
+               struct rlimit   rlim;
 
                libxfs_icache_purge();
                libxfs_bcache_purge();
@@ -560,27 +572,46 @@ main(int argc, char **argv)
                cache_destroy(libxfs_bcache);
 
                mem_used = (mp->m_sb.sb_icount >> (10 - 2)) +
-                                       (mp->m_sb.sb_dblocks >> (10 + 1));
-               phys_mem = libxfs_physmem() * 3 / 4;
+                                       (mp->m_sb.sb_dblocks >> (10 + 1)) +
+                                       50000;  /* rough estimate of 50MB overhead */
+               max_mem = max_mem_specified ? max_mem_specified * 1024 :
+                                               libxfs_physmem() * 3 / 4;
+
+               if (getrlimit(RLIMIT_AS, &rlim) != -1 &&
+                                       rlim.rlim_cur != RLIM_INFINITY) {
+                       rlim.rlim_cur = rlim.rlim_max;
+                       setrlimit(RLIMIT_AS, &rlim);
+                       /* use approximately 80% of rlimit to avoid overrun */
+                       max_mem = MIN(max_mem, rlim.rlim_cur / 1280);
+               } else
+                       max_mem = MIN(max_mem, (LONG_MAX >> 10) + 1);
 
                if (verbose > 1)
-                       do_log(_("        - icount = %llu, imem = %lu, "
-                               "dblock = %llu, dmem = %lu\n"),
-                               mp->m_sb.sb_icount, mp->m_sb.sb_icount >> (10 - 2),
-                               mp->m_sb.sb_dblocks, mp->m_sb.sb_dblocks >> (10 + 1));
-
-               if (phys_mem <= mem_used) {
+                       do_log(_("        - max_mem = %lu, icount = %llu, "
+                               "imem = %llu, dblock = %llu, dmem = %llu\n"),
+                               max_mem, mp->m_sb.sb_icount,
+                               mp->m_sb.sb_icount >> (10 - 2),
+                               mp->m_sb.sb_dblocks,
+                               mp->m_sb.sb_dblocks >> (10 + 1));
+
+               if (max_mem <= mem_used) {
                        /*
                         * Turn off prefetch and minimise libxfs cache if
                         * physical memory is deemed insufficient
                         */
+                       if (max_mem_specified)
+                               do_abort(_("Required memory for repair is "
+                                       "greater that the maximum specified "
+                                       "with the -m option. Please increase "
+                                       "it to at least %lu.\n"),
+                                       mem_used / 1024);
                        do_prefetch = 0;
                        libxfs_bhash_size = 64;
                } else {
-                       phys_mem -= mem_used;
-                       if (phys_mem >= (1 << 30))
-                               phys_mem = 1 << 30;
-                       libxfs_bhash_size = phys_mem / (HASH_CACHE_RATIO *
+                       max_mem -= mem_used;
+                       if (max_mem >= (1 << 30))
+                               max_mem = 1 << 30;
+                       libxfs_bhash_size = max_mem / (HASH_CACHE_RATIO *
                                        (mp->m_inode_cluster_size >> 10));
                        if (libxfs_bhash_size < 512)
                                libxfs_bhash_size = 512;