dm: fix deadlock when swapping to encrypted device

commit a666e5c05e7c4aaabb2c5d58117b0946803d03d2 upstream.

The system would deadlock when swapping to a dm-crypt device. The reason
is that for each incoming write bio, dm-crypt allocates memory that holds
encrypted data. These excessive allocations exhaust all the memory and the
result is either deadlock or OOM trigger.

This patch limits the number of in-flight swap bios, so that the memory
consumed by dm-crypt is limited. The limit is enforced if the target set
the "limit_swap_bios" variable and if the bio has REQ_SWAP set.

Non-swap bios are not affected becuase taking the semaphore would cause
performance degradation.

This is similar to request-based drivers - they will also block when the
number of requests is over the limit.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Cc: stable@vger.kernel.org
Signed-off-by: Mike Snitzer <snitzer@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index c4ef1fceead6ee1ba83bfb0e54f2eefae26bd48b..3fea121fcbcf933aa2cbea9c7d4ec4444910dbde 100644 (file)
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -106,6 +106,10 @@ struct mapped_device {
 
        struct block_device *bdev;
 
+       int swap_bios;
+       struct semaphore swap_bios_semaphore;
+       struct mutex swap_bios_lock;
+
        struct dm_stats stats;
 
        /* for blk-mq request-based DM support */

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 1af82fbbac0c42efcc8fb1f052c60628a75a4651..d85648b9c247acb6e3700beafeaeb50354e404fe 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -2737,6 +2737,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
        wake_up_process(cc->write_thread);
 
        ti->num_flush_bios = 1;
+       ti->limit_swap_bios = true;
 
        return 0;
 

diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index c6ce42daff27bb8957ae0e541236c3b091c2621b..de32f8553735f45a734a7be21f9a1f4c9ecb07c2 100644 (file)
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -146,6 +146,16 @@ EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
 #define DM_NUMA_NODE NUMA_NO_NODE
 static int dm_numa_node = DM_NUMA_NODE;
 
+#define DEFAULT_SWAP_BIOS      (8 * 1048576 / PAGE_SIZE)
+static int swap_bios = DEFAULT_SWAP_BIOS;
+static int get_swap_bios(void)
+{
+       int latch = READ_ONCE(swap_bios);
+       if (unlikely(latch <= 0))
+               latch = DEFAULT_SWAP_BIOS;
+       return latch;
+}
+
 /*
  * For mempools pre-allocation at the table loading time.
  */
@@ -972,6 +982,11 @@ void disable_write_zeroes(struct mapped_device *md)
        limits->max_write_zeroes_sectors = 0;
 }
 
+static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
+{
+       return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
+}
+
 static void clone_endio(struct bio *bio)
 {
        blk_status_t error = bio->bi_status;
@@ -1009,6 +1024,11 @@ static void clone_endio(struct bio *bio)
                }
        }
 
+       if (unlikely(swap_bios_limit(tio->ti, bio))) {
+               struct mapped_device *md = io->md;
+               up(&md->swap_bios_semaphore);
+       }
+
        free_tio(tio);
        dec_pending(io, error);
 }
@@ -1263,6 +1283,22 @@ void dm_remap_zone_report(struct dm_target *ti, sector_t start,
 }
 EXPORT_SYMBOL_GPL(dm_remap_zone_report);
 
+static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
+{
+       mutex_lock(&md->swap_bios_lock);
+       while (latch < md->swap_bios) {
+               cond_resched();
+               down(&md->swap_bios_semaphore);
+               md->swap_bios--;
+       }
+       while (latch > md->swap_bios) {
+               cond_resched();
+               up(&md->swap_bios_semaphore);
+               md->swap_bios++;
+       }
+       mutex_unlock(&md->swap_bios_lock);
+}
+
 static blk_qc_t __map_bio(struct dm_target_io *tio)
 {
        int r;
@@ -1283,6 +1319,14 @@ static blk_qc_t __map_bio(struct dm_target_io *tio)
        atomic_inc(&io->io_count);
        sector = clone->bi_iter.bi_sector;
 
+       if (unlikely(swap_bios_limit(ti, clone))) {
+               struct mapped_device *md = io->md;
+               int latch = get_swap_bios();
+               if (unlikely(latch != md->swap_bios))
+                       __set_swap_bios_limit(md, latch);
+               down(&md->swap_bios_semaphore);
+       }
+
        r = ti->type->map(ti, clone);
        switch (r) {
        case DM_MAPIO_SUBMITTED:
@@ -1297,10 +1341,18 @@ static blk_qc_t __map_bio(struct dm_target_io *tio)
                        ret = generic_make_request(clone);
                break;
        case DM_MAPIO_KILL:
+               if (unlikely(swap_bios_limit(ti, clone))) {
+                       struct mapped_device *md = io->md;
+                       up(&md->swap_bios_semaphore);
+               }
                free_tio(tio);
                dec_pending(io, BLK_STS_IOERR);
                break;
        case DM_MAPIO_REQUEUE:
+               if (unlikely(swap_bios_limit(ti, clone))) {
+                       struct mapped_device *md = io->md;
+                       up(&md->swap_bios_semaphore);
+               }
                free_tio(tio);
                dec_pending(io, BLK_STS_DM_REQUEUE);
                break;
@@ -1894,6 +1946,7 @@ static void cleanup_mapped_device(struct mapped_device *md)
        mutex_destroy(&md->suspend_lock);
        mutex_destroy(&md->type_lock);
        mutex_destroy(&md->table_devices_lock);
+       mutex_destroy(&md->swap_bios_lock);
 
        dm_mq_cleanup_mapped_device(md);
 }
@@ -1963,6 +2016,10 @@ static struct mapped_device *alloc_dev(int minor)
        init_waitqueue_head(&md->eventq);
        init_completion(&md->kobj_holder.completion);
 
+       md->swap_bios = get_swap_bios();
+       sema_init(&md->swap_bios_semaphore, md->swap_bios);
+       mutex_init(&md->swap_bios_lock);
+
        md->disk->major = _major;
        md->disk->first_minor = minor;
        md->disk->fops = &dm_blk_dops;
@@ -3245,6 +3302,9 @@ MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
 module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
 MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
 
+module_param(swap_bios, int, S_IRUGO | S_IWUSR);
+MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
+
 MODULE_DESCRIPTION(DM_NAME " driver");
 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
 MODULE_LICENSE("GPL");

diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h
index e4e1f5c1f492908de697fb39ab4add8746d3e7e1..a53d7d2c2d95ce425bddb10fabfeff5c6eea1015 100644 (file)
--- a/include/linux/device-mapper.h
+++ b/include/linux/device-mapper.h
@@ -316,6 +316,11 @@ struct dm_target {
         * whether or not its underlying devices have support.
         */
        bool discards_supported:1;
+
+       /*
+        * Set if we need to limit the number of in-flight bios when swapping.
+        */
+       bool limit_swap_bios:1;
 };
 
 /* Each target can link one of these into the table */