libxfs: simulate system failure after a certain number of writes

Add an error injection knob so that we can simulate system failure after
a certain number of disk writes.  This knob is being added so that we
can check repair's behavior after an arbitrary number of tests.

Set LIBXFS_DEBUG_WRITE_CRASH={ddev,logdev,rtdev}=nn in the environment
to make libxfs SIGKILL itself after nn writes to the data, log, or rt
devices.  Note that this only applies to xfs_buf writes and zero_range.

Signed-off-by: Darrick J. Wong <djwong@kernel.org>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Signed-off-by: Eric Sandeen <sandeen@sandeen.net>

diff --git a/include/linux.h b/include/linux.h
index 03b3278bb895d13dc6e0f6ec80df0aa1b660d1b9..7bf59e07b61508252d5c6a480a09a1bd168e4509 100644 (file)
--- a/include/linux.h
+++ b/include/linux.h
@@ -31,6 +31,8 @@
 #ifdef OVERRIDE_SYSTEM_FSXATTR
 # undef fsxattr
 #endif
+#include <unistd.h>
+#include <assert.h>
 
 static __inline__ int xfsctl(const char *path, int fd, int cmd, void *p)
 {
@@ -186,6 +188,17 @@ platform_zero_range(
 #define platform_zero_range(fd, s, l)  (-EOPNOTSUPP)
 #endif
 
+/*
+ * Use SIGKILL to simulate an immediate program crash, without a chance to run
+ * atexit handlers.
+ */
+static inline void
+platform_crash(void)
+{
+       kill(getpid(), SIGKILL);
+       assert(0);
+}
+
 /*
  * Check whether we have to define FS_IOC_FS[GS]ETXATTR ourselves. These
  * are a copy of the definitions moved to linux/uapi/fs.h in the 4.5 kernel,

diff --git a/libxfs/init.c b/libxfs/init.c
index 8a8ce3c4b7fd38b582dece040ed62a7c4239d9ad..1ec837911df72cf80e38ef3348a0e736a0a521c3 100644 (file)
--- a/libxfs/init.c
+++ b/libxfs/init.c
@@ -590,7 +590,8 @@ out_unwind:
 static struct xfs_buftarg *
 libxfs_buftarg_alloc(
        struct xfs_mount        *mp,
-       dev_t                   dev)
+       dev_t                   dev,
+       unsigned long           write_fails)
 {
        struct xfs_buftarg      *btp;
 
@@ -603,10 +604,29 @@ libxfs_buftarg_alloc(
        btp->bt_mount = mp;
        btp->bt_bdev = dev;
        btp->flags = 0;
+       if (write_fails) {
+               btp->writes_left = write_fails;
+               btp->flags |= XFS_BUFTARG_INJECT_WRITE_FAIL;
+       }
+       pthread_mutex_init(&btp->lock, NULL);
 
        return btp;
 }
 
+enum libxfs_write_failure_nums {
+       WF_DATA = 0,
+       WF_LOG,
+       WF_RT,
+       WF_MAX_OPTS,
+};
+
+static char *wf_opts[] = {
+       [WF_DATA]               = "ddev",
+       [WF_LOG]                = "logdev",
+       [WF_RT]                 = "rtdev",
+       [WF_MAX_OPTS]           = NULL,
+};
+
 void
 libxfs_buftarg_init(
        struct xfs_mount        *mp,
@@ -614,6 +634,46 @@ libxfs_buftarg_init(
        dev_t                   logdev,
        dev_t                   rtdev)
 {
+       char                    *p = getenv("LIBXFS_DEBUG_WRITE_CRASH");
+       unsigned long           dfail = 0, lfail = 0, rfail = 0;
+
+       /* Simulate utility crash after a certain number of writes. */
+       while (p && *p) {
+               char *val;
+
+               switch (getsubopt(&p, wf_opts, &val)) {
+               case WF_DATA:
+                       if (!val) {
+                               fprintf(stderr,
+               _("ddev write fail requires a parameter\n"));
+                               exit(1);
+                       }
+                       dfail = strtoul(val, NULL, 0);
+                       break;
+               case WF_LOG:
+                       if (!val) {
+                               fprintf(stderr,
+               _("logdev write fail requires a parameter\n"));
+                               exit(1);
+                       }
+                       lfail = strtoul(val, NULL, 0);
+                       break;
+               case WF_RT:
+                       if (!val) {
+                               fprintf(stderr,
+               _("rtdev write fail requires a parameter\n"));
+                               exit(1);
+                       }
+                       rfail = strtoul(val, NULL, 0);
+                       break;
+               default:
+                       fprintf(stderr, _("unknown write fail type %s\n"),
+                                       val);
+                       exit(1);
+                       break;
+               }
+       }
+
        if (mp->m_ddev_targp) {
                /* should already have all buftargs initialised */
                if (mp->m_ddev_targp->bt_bdev != dev ||
@@ -647,12 +707,12 @@ libxfs_buftarg_init(
                return;
        }
 
-       mp->m_ddev_targp = libxfs_buftarg_alloc(mp, dev);
+       mp->m_ddev_targp = libxfs_buftarg_alloc(mp, dev, dfail);
        if (!logdev || logdev == dev)
                mp->m_logdev_targp = mp->m_ddev_targp;
        else
-               mp->m_logdev_targp = libxfs_buftarg_alloc(mp, logdev);
-       mp->m_rtdev_targp = libxfs_buftarg_alloc(mp, rtdev);
+               mp->m_logdev_targp = libxfs_buftarg_alloc(mp, logdev, lfail);
+       mp->m_rtdev_targp = libxfs_buftarg_alloc(mp, rtdev, rfail);
 }
 
 /*

diff --git a/libxfs/libxfs_io.h b/libxfs/libxfs_io.h
index c80e2d59ad2184254f409d0defee3c4e580a7d65..3cc4f4ee87eeda816278a18465924006afa53481 100644 (file)
--- a/libxfs/libxfs_io.h
+++ b/libxfs/libxfs_io.h
@@ -22,6 +22,8 @@ struct xfs_perag;
  */
 struct xfs_buftarg {
        struct xfs_mount        *bt_mount;
+       pthread_mutex_t         lock;
+       unsigned long           writes_left;
        dev_t                   bt_bdev;
        unsigned int            flags;
 };
@@ -30,6 +32,23 @@ struct xfs_buftarg {
 #define XFS_BUFTARG_LOST_WRITE         (1 << 0)
 /* A dirty buffer failed the write verifier. */
 #define XFS_BUFTARG_CORRUPT_WRITE      (1 << 1)
+/* Simulate failure after a certain number of writes. */
+#define XFS_BUFTARG_INJECT_WRITE_FAIL  (1 << 2)
+
+/* Simulate the system crashing after a certain number of writes. */
+static inline void
+xfs_buftarg_trip_write(
+       struct xfs_buftarg      *btp)
+{
+       if (!(btp->flags & XFS_BUFTARG_INJECT_WRITE_FAIL))
+               return;
+
+       pthread_mutex_lock(&btp->lock);
+       btp->writes_left--;
+       if (!btp->writes_left)
+               platform_crash();
+       pthread_mutex_unlock(&btp->lock);
+}
 
 extern void    libxfs_buftarg_init(struct xfs_mount *mp, dev_t ddev,
                                    dev_t logdev, dev_t rtdev);

diff --git a/libxfs/rdwr.c b/libxfs/rdwr.c
index ca272387a76bba9b355eb7351f616fadb41654db..fd456d6b4678264f932953ca2961cf0a6c8a06f6 100644 (file)
--- a/libxfs/rdwr.c
+++ b/libxfs/rdwr.c
@@ -74,8 +74,10 @@ libxfs_device_zero(struct xfs_buftarg *btp, xfs_daddr_t start, uint len)
        /* try to use special zeroing methods, fall back to writes if needed */
        len_bytes = LIBXFS_BBTOOFF64(len);
        error = platform_zero_range(fd, start_offset, len_bytes);
-       if (!error)
+       if (!error) {
+               xfs_buftarg_trip_write(btp);
                return 0;
+       }
 
        zsize = min(BDSTRAT_SIZE, BBTOB(len));
        if ((z = memalign(libxfs_device_alignment(), zsize)) == NULL) {
@@ -105,6 +107,7 @@ libxfs_device_zero(struct xfs_buftarg *btp, xfs_daddr_t start, uint len)
                                progname, __FUNCTION__);
                        exit(1);
                }
+               xfs_buftarg_trip_write(btp);
                offset += bytes;
        }
        free(z);
@@ -860,6 +863,7 @@ libxfs_bwrite(
        } else {
                bp->b_flags |= LIBXFS_B_UPTODATE;
                bp->b_flags &= ~(LIBXFS_B_DIRTY | LIBXFS_B_UNCHECKED);
+               xfs_buftarg_trip_write(bp->b_target);
        }
        return bp->b_error;
 }