block: Allow mapping of vmalloc-ed buffers

[thirdparty/kernel/stable.git] / block / bio.c

diff --git a/block/bio.c b/block/bio.c
index a3c80a6c1fe51d4c18aa52fb5e508d958c6ba16d..e73a2a4c01cdf9de6309de8a079fb505dd7db37e 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -29,6 +29,7 @@
 #include <linux/workqueue.h>
 #include <linux/cgroup.h>
 #include <linux/blk-cgroup.h>
+#include <linux/highmem.h>
 
 #include <trace/events/block.h>
 #include "blk.h"
@@ -1475,8 +1476,22 @@ void bio_unmap_user(struct bio *bio)
        bio_put(bio);
 }
 
+static void bio_invalidate_vmalloc_pages(struct bio *bio)
+{
+#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
+       if (bio->bi_private && !op_is_write(bio_op(bio))) {
+               unsigned long i, len = 0;
+
+               for (i = 0; i < bio->bi_vcnt; i++)
+                       len += bio->bi_io_vec[i].bv_len;
+               invalidate_kernel_vmap_range(bio->bi_private, len);
+       }
+#endif
+}
+
 static void bio_map_kern_endio(struct bio *bio)
 {
+       bio_invalidate_vmalloc_pages(bio);
        bio_put(bio);
 }
 
@@ -1497,6 +1512,8 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
        unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
        unsigned long start = kaddr >> PAGE_SHIFT;
        const int nr_pages = end - start;
+       bool is_vmalloc = is_vmalloc_addr(data);
+       struct page *page;
        int offset, i;
        struct bio *bio;
 
@@ -1504,6 +1521,11 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
        if (!bio)
                return ERR_PTR(-ENOMEM);
 
+       if (is_vmalloc) {
+               flush_kernel_vmap_range(data, len);
+               bio->bi_private = data;
+       }
+
        offset = offset_in_page(kaddr);
        for (i = 0; i < nr_pages; i++) {
                unsigned int bytes = PAGE_SIZE - offset;
@@ -1514,7 +1536,11 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
                if (bytes > len)
                        bytes = len;
 
-               if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
+               if (!is_vmalloc)
+                       page = virt_to_page(data);
+               else
+                       page = vmalloc_to_page(data);
+               if (bio_add_pc_page(q, bio, page, bytes,
                                    offset) < bytes) {
                        /* we don't support partial mappings */
                        bio_put(bio);