[thirdparty/kernel/stable.git] / block / bounce.c

// SPDX-License-Identifier: GPL-2.0
/* bounce buffer handling for block devices
 *
 * - Split from highmem.c
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/mm.h>
#include <linux/export.h>
#include <linux/swap.h>
#include <linux/gfp.h>
#include <linux/bio.h>
#include <linux/pagemap.h>
#include <linux/mempool.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/init.h>
#include <linux/hash.h>
#include <linux/highmem.h>
#include <linux/memblock.h>
#include <linux/printk.h>
#include <asm/tlbflush.h>

#include <trace/events/block.h>
#include "blk.h"

#define POOL_SIZE	64
#define ISA_POOL_SIZE	16

static struct bio_set bounce_bio_set, bounce_bio_split;
static mempool_t page_pool, isa_page_pool;

static void init_bounce_bioset(void)
{
	static bool bounce_bs_setup;
	int ret;

	if (bounce_bs_setup)
		return;

	ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
	BUG_ON(ret);
	if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE))
		BUG_ON(1);

	ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0);
	BUG_ON(ret);
	bounce_bs_setup = true;
}

#if defined(CONFIG_HIGHMEM)
static __init int init_emergency_pool(void)
{
	int ret;
#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
	if (max_pfn <= max_low_pfn)
		return 0;
#endif

	ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0);
	BUG_ON(ret);
	pr_info("pool size: %d pages\n", POOL_SIZE);

	init_bounce_bioset();
	return 0;
}

__initcall(init_emergency_pool);
#endif

#ifdef CONFIG_HIGHMEM
/*
 * highmem version, map in to vec
 */
static void bounce_copy_vec(struct bio_vec *to, unsigned char *vfrom)
{
	unsigned char *vto;

	vto = kmap_atomic(to->bv_page);
	memcpy(vto + to->bv_offset, vfrom, to->bv_len);
	kunmap_atomic(vto);
}

#else /* CONFIG_HIGHMEM */

#define bounce_copy_vec(to, vfrom)	\
	memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)

#endif /* CONFIG_HIGHMEM */

/*
 * allocate pages in the DMA region for the ISA pool
 */
static void *mempool_alloc_pages_isa(gfp_t gfp_mask, void *data)
{
	return mempool_alloc_pages(gfp_mask | GFP_DMA, data);
}

static DEFINE_MUTEX(isa_mutex);

/*
 * gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
 * as the max address, so check if the pool has already been created.
 */
int init_emergency_isa_pool(void)
{
	int ret;

	mutex_lock(&isa_mutex);

	if (mempool_initialized(&isa_page_pool)) {
		mutex_unlock(&isa_mutex);
		return 0;
	}

	ret = mempool_init(&isa_page_pool, ISA_POOL_SIZE, mempool_alloc_pages_isa,
			   mempool_free_pages, (void *) 0);
	BUG_ON(ret);

	pr_info("isa pool size: %d pages\n", ISA_POOL_SIZE);
	init_bounce_bioset();
	mutex_unlock(&isa_mutex);
	return 0;
}

/*
 * Simple bounce buffer support for highmem pages. Depending on the
 * queue gfp mask set, *to may or may not be a highmem page. kmap it
 * always, it will do the Right Thing
 */
static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
{
	unsigned char *vfrom;
	struct bio_vec tovec, fromvec;
	struct bvec_iter iter;
	/*
	 * The bio of @from is created by bounce, so we can iterate
	 * its bvec from start to end, but the @from->bi_iter can't be
	 * trusted because it might be changed by splitting.
	 */
	struct bvec_iter from_iter = BVEC_ITER_ALL_INIT;

	bio_for_each_segment(tovec, to, iter) {
		fromvec = bio_iter_iovec(from, from_iter);
		if (tovec.bv_page != fromvec.bv_page) {
			/*
			 * fromvec->bv_offset and fromvec->bv_len might have
			 * been modified by the block layer, so use the original
			 * copy, bounce_copy_vec already uses tovec->bv_len
			 */
			vfrom = page_address(fromvec.bv_page) +
				tovec.bv_offset;

			bounce_copy_vec(&tovec, vfrom);
			flush_dcache_page(tovec.bv_page);
		}
		bio_advance_iter(from, &from_iter, tovec.bv_len);
	}
}

static void bounce_end_io(struct bio *bio, mempool_t *pool)
{
	struct bio *bio_orig = bio->bi_private;
	struct bio_vec *bvec, orig_vec;
	struct bvec_iter orig_iter = bio_orig->bi_iter;
	struct bvec_iter_all iter_all;

	/*
	 * free up bounce indirect pages used
	 */
	bio_for_each_segment_all(bvec, bio, iter_all) {
		orig_vec = bio_iter_iovec(bio_orig, orig_iter);
		if (bvec->bv_page != orig_vec.bv_page) {
			dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
			mempool_free(bvec->bv_page, pool);
		}
		bio_advance_iter(bio_orig, &orig_iter, orig_vec.bv_len);
	}

	bio_orig->bi_status = bio->bi_status;
	bio_endio(bio_orig);
	bio_put(bio);
}

static void bounce_end_io_write(struct bio *bio)
{
	bounce_end_io(bio, &page_pool);
}

static void bounce_end_io_write_isa(struct bio *bio)
{

	bounce_end_io(bio, &isa_page_pool);
}

static void __bounce_end_io_read(struct bio *bio, mempool_t *pool)
{
	struct bio *bio_orig = bio->bi_private;

	if (!bio->bi_status)
		copy_to_high_bio_irq(bio_orig, bio);

	bounce_end_io(bio, pool);
}

static void bounce_end_io_read(struct bio *bio)
{
	__bounce_end_io_read(bio, &page_pool);
}

static void bounce_end_io_read_isa(struct bio *bio)
{
	__bounce_end_io_read(bio, &isa_page_pool);
}

static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask,
		struct bio_set *bs)
{
	struct bvec_iter iter;
	struct bio_vec bv;
	struct bio *bio;

	/*
	 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
	 * bio_src->bi_io_vec to bio->bi_io_vec.
	 *
	 * We can't do that anymore, because:
	 *
	 *  - The point of cloning the biovec is to produce a bio with a biovec
	 *    the caller can modify: bi_idx and bi_bvec_done should be 0.
	 *
	 *  - The original bio could've had more than BIO_MAX_PAGES biovecs; if
	 *    we tried to clone the whole thing bio_alloc_bioset() would fail.
	 *    But the clone should succeed as long as the number of biovecs we
	 *    actually need to allocate is fewer than BIO_MAX_PAGES.
	 *
	 *  - Lastly, bi_vcnt should not be looked at or relied upon by code
	 *    that does not own the bio - reason being drivers don't use it for
	 *    iterating over the biovec anymore, so expecting it to be kept up
	 *    to date (i.e. for clones that share the parent biovec) is just
	 *    asking for trouble and would force extra work on
	 *    __bio_clone_fast() anyways.
	 */

	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
	if (!bio)
		return NULL;
	bio->bi_disk		= bio_src->bi_disk;
	bio->bi_opf		= bio_src->bi_opf;
	bio->bi_ioprio		= bio_src->bi_ioprio;
	bio->bi_write_hint	= bio_src->bi_write_hint;
	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;
	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;

	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
	case REQ_OP_SECURE_ERASE:
	case REQ_OP_WRITE_ZEROES:
		break;
	case REQ_OP_WRITE_SAME:
		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
		break;
	default:
		bio_for_each_segment(bv, bio_src, iter)
			bio->bi_io_vec[bio->bi_vcnt++] = bv;
		break;
	}

	if (bio_integrity(bio_src)) {
		int ret;

		ret = bio_integrity_clone(bio, bio_src, gfp_mask);
		if (ret < 0) {
			bio_put(bio);
			return NULL;
		}
	}

	bio_clone_blkg_association(bio, bio_src);
	blkcg_bio_issue_init(bio);

	return bio;
}

static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
			       mempool_t *pool)
{
	struct bio *bio;
	int rw = bio_data_dir(*bio_orig);
	struct bio_vec *to, from;
	struct bvec_iter iter;
	unsigned i = 0;
	bool bounce = false;
	int sectors = 0;
	bool passthrough = bio_is_passthrough(*bio_orig);

	bio_for_each_segment(from, *bio_orig, iter) {
		if (i++ < BIO_MAX_PAGES)
			sectors += from.bv_len >> 9;
		if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn)
			bounce = true;
	}
	if (!bounce)
		return;

	if (!passthrough && sectors < bio_sectors(*bio_orig)) {
		bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
		bio_chain(bio, *bio_orig);
		generic_make_request(*bio_orig);
		*bio_orig = bio;
	}
	bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL :
			&bounce_bio_set);

	/*
	 * Bvec table can't be updated by bio_for_each_segment_all(),
	 * so retrieve bvec from the table directly. This way is safe
	 * because the 'bio' is single-page bvec.
	 */
	for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {
		struct page *page = to->bv_page;

		if (page_to_pfn(page) <= q->limits.bounce_pfn)
			continue;

		to->bv_page = mempool_alloc(pool, q->bounce_gfp);
		inc_zone_page_state(to->bv_page, NR_BOUNCE);

		if (rw == WRITE) {
			char *vto, *vfrom;

			flush_dcache_page(page);

			vto = page_address(to->bv_page) + to->bv_offset;
			vfrom = kmap_atomic(page) + to->bv_offset;
			memcpy(vto, vfrom, to->bv_len);
			kunmap_atomic(vfrom);
		}
	}

	trace_block_bio_bounce(q, *bio_orig);

	bio->bi_flags |= (1 << BIO_BOUNCED);

	if (pool == &page_pool) {
		bio->bi_end_io = bounce_end_io_write;
		if (rw == READ)
			bio->bi_end_io = bounce_end_io_read;
	} else {
		bio->bi_end_io = bounce_end_io_write_isa;
		if (rw == READ)
			bio->bi_end_io = bounce_end_io_read_isa;
	}

	bio->bi_private = *bio_orig;
	*bio_orig = bio;
}

void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
{
	mempool_t *pool;

	/*
	 * Data-less bio, nothing to bounce
	 */
	if (!bio_has_data(*bio_orig))
		return;

	/*
	 * for non-isa bounce case, just check if the bounce pfn is equal
	 * to or bigger than the highest pfn in the system -- in that case,
	 * don't waste time iterating over bio segments
	 */
	if (!(q->bounce_gfp & GFP_DMA)) {
		if (q->limits.bounce_pfn >= blk_max_pfn)
			return;
		pool = &page_pool;
	} else {
		BUG_ON(!mempool_initialized(&isa_page_pool));
		pool = &isa_page_pool;
	}

	/*
	 * slow path
	 */
	__blk_queue_bounce(q, bio_orig, pool);
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
831058de DH	2	/* bounce buffer handling for block devices
	3	*
	4	* - Split from highmem.c
	5	*/
	6
b1de0d13 MH	7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
b1de0d13 MH	8
831058de	9	#include <linux/mm.h>
b95f1b31	10	#include <linux/export.h>
831058de	11	#include <linux/swap.h>
5a0e3ad6	12	#include <linux/gfp.h>
831058de DH	13	#include <linux/bio.h>
	14	#include <linux/pagemap.h>
	15	#include <linux/mempool.h>
	16	#include <linux/blkdev.h>
66114cad	17	#include <linux/backing-dev.h>
831058de DH	18	#include <linux/init.h>
	19	#include <linux/hash.h>
	20	#include <linux/highmem.h>
57c8a661	21	#include <linux/memblock.h>
b1de0d13	22	#include <linux/printk.h>
831058de DH	23	#include <asm/tlbflush.h>
831058de DH	24
55782138	25	#include <trace/events/block.h>
3bce016a	26	#include "blk.h"
55782138	27
831058de DH	28	#define POOL_SIZE 64
	29	#define ISA_POOL_SIZE 16
	30
338aa96d KO	31	static struct bio_set bounce_bio_set, bounce_bio_split;
338aa96d KO	32	static mempool_t page_pool, isa_page_pool;
831058de	33
52990a5f JA	34	static void init_bounce_bioset(void)
	35	{
	36	static bool bounce_bs_setup;
	37	int ret;
	38
	39	if (bounce_bs_setup)
	40	return;
	41
	42	ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
	43	BUG_ON(ret);
	44	if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE))
	45	BUG_ON(1);
	46
	47	ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0);
	48	BUG_ON(ret);
	49	bounce_bs_setup = true;
	50	}
	51
a687a533	52	#if defined(CONFIG_HIGHMEM)
831058de DH	53	static __init int init_emergency_pool(void)
831058de DH	54	{
338aa96d	55	int ret;
f1006257	56	#if defined(CONFIG_HIGHMEM) && !defined(CONFIG_MEMORY_HOTPLUG)
3bcfeaf9	57	if (max_pfn <= max_low_pfn)
831058de	58	return 0;
3bcfeaf9	59	#endif
831058de	60
338aa96d KO	61	ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0);
338aa96d KO	62	BUG_ON(ret);
b1de0d13	63	pr_info("pool size: %d pages\n", POOL_SIZE);
831058de	64
52990a5f	65	init_bounce_bioset();
831058de DH	66	return 0;
	67	}
	68
	69	__initcall(init_emergency_pool);
f1006257	70	#endif
831058de	71
f1006257	72	#ifdef CONFIG_HIGHMEM
831058de DH	73	/*
	74	* highmem version, map in to vec
	75	*/
	76	static void bounce_copy_vec(struct bio_vec to, unsigned char vfrom)
	77	{
831058de DH	78	unsigned char *vto;
831058de DH	79
9b04c5fe	80	vto = kmap_atomic(to->bv_page);
831058de	81	memcpy(vto + to->bv_offset, vfrom, to->bv_len);
9b04c5fe	82	kunmap_atomic(vto);
831058de DH	83	}
	84
	85	#else /* CONFIG_HIGHMEM */
	86
	87	#define bounce_copy_vec(to, vfrom) \
	88	memcpy(page_address((to)->bv_page) + (to)->bv_offset, vfrom, (to)->bv_len)
	89
	90	#endif /* CONFIG_HIGHMEM */
	91
	92	/*
	93	* allocate pages in the DMA region for the ISA pool
	94	*/
	95	static void mempool_alloc_pages_isa(gfp_t gfp_mask, void data)
	96	{
	97	return mempool_alloc_pages(gfp_mask \| GFP_DMA, data);
	98	}
	99
52990a5f JA	100	static DEFINE_MUTEX(isa_mutex);
52990a5f JA	101
831058de DH	102	/*
	103	* gets called "every" time someone init's a queue with BLK_BOUNCE_ISA
	104	* as the max address, so check if the pool has already been created.
	105	*/
	106	int init_emergency_isa_pool(void)
	107	{
338aa96d KO	108	int ret;
338aa96d KO	109
52990a5f JA	110	mutex_lock(&isa_mutex);
	111
	112	if (mempool_initialized(&isa_page_pool)) {
	113	mutex_unlock(&isa_mutex);
831058de	114	return 0;
52990a5f	115	}
831058de	116
338aa96d KO	117	ret = mempool_init(&isa_page_pool, ISA_POOL_SIZE, mempool_alloc_pages_isa,
	118	mempool_free_pages, (void *) 0);
	119	BUG_ON(ret);
831058de	120
b1de0d13	121	pr_info("isa pool size: %d pages\n", ISA_POOL_SIZE);
52990a5f JA	122	init_bounce_bioset();
52990a5f JA	123	mutex_unlock(&isa_mutex);
831058de DH	124	return 0;
	125	}
	126
	127	/*
	128	* Simple bounce buffer support for highmem pages. Depending on the
	129	* queue gfp mask set, *to may or may not be a highmem page. kmap it
	130	* always, it will do the Right Thing
	131	*/
	132	static void copy_to_high_bio_irq(struct bio to, struct bio from)
	133	{
	134	unsigned char *vfrom;
3c892a09	135	struct bio_vec tovec, fromvec;
7988613b	136	struct bvec_iter iter;
3c892a09 ML	137	/*
	138	* The bio of @from is created by bounce, so we can iterate
	139	* its bvec from start to end, but the @from->bi_iter can't be
	140	* trusted because it might be changed by splitting.
	141	*/
	142	struct bvec_iter from_iter = BVEC_ITER_ALL_INIT;
7988613b KO	143
7988613b KO	144	bio_for_each_segment(tovec, to, iter) {
3c892a09 ML	145	fromvec = bio_iter_iovec(from, from_iter);
3c892a09 ML	146	if (tovec.bv_page != fromvec.bv_page) {
7988613b KO	147	/*
	148	* fromvec->bv_offset and fromvec->bv_len might have
	149	* been modified by the block layer, so use the original
	150	* copy, bounce_copy_vec already uses tovec->bv_len
	151	*/
3c892a09	152	vfrom = page_address(fromvec.bv_page) +
7988613b KO	153	tovec.bv_offset;
	154
	155	bounce_copy_vec(&tovec, vfrom);
	156	flush_dcache_page(tovec.bv_page);
	157	}
3c892a09	158	bio_advance_iter(from, &from_iter, tovec.bv_len);
831058de DH	159	}
	160	}
	161
4246a0b6	162	static void bounce_end_io(struct bio bio, mempool_t pool)
831058de DH	163	{
831058de DH	164	struct bio *bio_orig = bio->bi_private;
7891f05c	165	struct bio_vec *bvec, orig_vec;
7891f05c	166	struct bvec_iter orig_iter = bio_orig->bi_iter;
6dc4f100	167	struct bvec_iter_all iter_all;
831058de	168
831058de DH	169	/*
	170	* free up bounce indirect pages used
	171	*/
2b070cfe	172	bio_for_each_segment_all(bvec, bio, iter_all) {
7891f05c ML	173	orig_vec = bio_iter_iovec(bio_orig, orig_iter);
	174	if (bvec->bv_page != orig_vec.bv_page) {
	175	dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
	176	mempool_free(bvec->bv_page, pool);
	177	}
	178	bio_advance_iter(bio_orig, &orig_iter, orig_vec.bv_len);
831058de DH	179	}
831058de DH	180
4e4cbee9	181	bio_orig->bi_status = bio->bi_status;
4246a0b6	182	bio_endio(bio_orig);
831058de DH	183	bio_put(bio);
	184	}
	185
4246a0b6	186	static void bounce_end_io_write(struct bio *bio)
831058de	187	{
338aa96d	188	bounce_end_io(bio, &page_pool);
831058de DH	189	}
831058de DH	190
4246a0b6	191	static void bounce_end_io_write_isa(struct bio *bio)
831058de	192	{
831058de	193
338aa96d	194	bounce_end_io(bio, &isa_page_pool);
831058de DH	195	}
831058de DH	196
4246a0b6	197	static void __bounce_end_io_read(struct bio bio, mempool_t pool)
831058de DH	198	{
	199	struct bio *bio_orig = bio->bi_private;
	200
4e4cbee9	201	if (!bio->bi_status)
831058de DH	202	copy_to_high_bio_irq(bio_orig, bio);
831058de DH	203
4246a0b6	204	bounce_end_io(bio, pool);
831058de DH	205	}
831058de DH	206
4246a0b6	207	static void bounce_end_io_read(struct bio *bio)
831058de	208	{
338aa96d	209	__bounce_end_io_read(bio, &page_pool);
831058de DH	210	}
831058de DH	211
4246a0b6	212	static void bounce_end_io_read_isa(struct bio *bio)
831058de	213	{
338aa96d	214	__bounce_end_io_read(bio, &isa_page_pool);
831058de DH	215	}
831058de DH	216
c55183c9 CH	217	static struct bio bounce_clone_bio(struct bio bio_src, gfp_t gfp_mask,
	218	struct bio_set *bs)
	219	{
	220	struct bvec_iter iter;
	221	struct bio_vec bv;
	222	struct bio *bio;
	223
	224	/*
	225	* Pre immutable biovecs, __bio_clone() used to just do a memcpy from
	226	* bio_src->bi_io_vec to bio->bi_io_vec.
	227	*
	228	* We can't do that anymore, because:
	229	*
	230	* - The point of cloning the biovec is to produce a bio with a biovec
	231	* the caller can modify: bi_idx and bi_bvec_done should be 0.
	232	*
	233	* - The original bio could've had more than BIO_MAX_PAGES biovecs; if
	234	* we tried to clone the whole thing bio_alloc_bioset() would fail.
	235	* But the clone should succeed as long as the number of biovecs we
	236	* actually need to allocate is fewer than BIO_MAX_PAGES.
	237	*
	238	* - Lastly, bi_vcnt should not be looked at or relied upon by code
	239	* that does not own the bio - reason being drivers don't use it for
	240	* iterating over the biovec anymore, so expecting it to be kept up
	241	* to date (i.e. for clones that share the parent biovec) is just
	242	* asking for trouble and would force extra work on
	243	* __bio_clone_fast() anyways.
	244	*/
	245
	246	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
	247	if (!bio)
	248	return NULL;
	249	bio->bi_disk = bio_src->bi_disk;
	250	bio->bi_opf = bio_src->bi_opf;
ca474b73	251	bio->bi_ioprio = bio_src->bi_ioprio;
c55183c9 CH	252	bio->bi_write_hint = bio_src->bi_write_hint;
	253	bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
	254	bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
	255
	256	switch (bio_op(bio)) {
	257	case REQ_OP_DISCARD:
	258	case REQ_OP_SECURE_ERASE:
	259	case REQ_OP_WRITE_ZEROES:
	260	break;
	261	case REQ_OP_WRITE_SAME:
	262	bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
	263	break;
	264	default:
	265	bio_for_each_segment(bv, bio_src, iter)
	266	bio->bi_io_vec[bio->bi_vcnt++] = bv;
	267	break;
	268	}
	269
	270	if (bio_integrity(bio_src)) {
	271	int ret;
	272
	273	ret = bio_integrity_clone(bio, bio_src, gfp_mask);
	274	if (ret < 0) {
	275	bio_put(bio);
	276	return NULL;
	277	}
	278	}
	279
db6638d7	280	bio_clone_blkg_association(bio, bio_src);
e439bedf	281	blkcg_bio_issue_init(bio);
5bf9a1f3	282
c55183c9 CH	283	return bio;
	284	}
	285
165125e1	286	static void __blk_queue_bounce(struct request_queue q, struct bio *bio_orig,
a3ad0a9d	287	mempool_t *pool)
831058de	288	{
6bc454d1 KO	289	struct bio *bio;
6bc454d1 KO	290	int rw = bio_data_dir(*bio_orig);
7988613b KO	291	struct bio_vec *to, from;
7988613b KO	292	struct bvec_iter iter;
a8821f3f N	293	unsigned i = 0;
	294	bool bounce = false;
	295	int sectors = 0;
14cb0dc6	296	bool passthrough = bio_is_passthrough(*bio_orig);
831058de	297
a8821f3f N	298	bio_for_each_segment(from, *bio_orig, iter) {
	299	if (i++ < BIO_MAX_PAGES)
	300	sectors += from.bv_len >> 9;
1c4bc3ab	301	if (page_to_pfn(from.bv_page) > q->limits.bounce_pfn)
a8821f3f N	302	bounce = true;
	303	}
	304	if (!bounce)
	305	return;
	306
14cb0dc6	307	if (!passthrough && sectors < bio_sectors(*bio_orig)) {
338aa96d	308	bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
a8821f3f N	309	bio_chain(bio, *bio_orig);
	310	generic_make_request(*bio_orig);
	311	*bio_orig = bio;
	312	}
c55183c9	313	bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL :
338aa96d	314	&bounce_bio_set);
831058de	315
8f4e80da ML	316	/*
	317	* Bvec table can't be updated by bio_for_each_segment_all(),
	318	* so retrieve bvec from the table directly. This way is safe
	319	* because the 'bio' is single-page bvec.
	320	*/
	321	for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {
6bc454d1	322	struct page *page = to->bv_page;
f735b5ee	323
1c4bc3ab	324	if (page_to_pfn(page) <= q->limits.bounce_pfn)
6bc454d1	325	continue;
831058de	326
6bc454d1	327	to->bv_page = mempool_alloc(pool, q->bounce_gfp);
393a3397	328	inc_zone_page_state(to->bv_page, NR_BOUNCE);
831058de DH	329
	330	if (rw == WRITE) {
	331	char vto, vfrom;
	332
6bc454d1 KO	333	flush_dcache_page(page);
6bc454d1 KO	334
831058de	335	vto = page_address(to->bv_page) + to->bv_offset;
6bc454d1	336	vfrom = kmap_atomic(page) + to->bv_offset;
831058de	337	memcpy(vto, vfrom, to->bv_len);
6bc454d1	338	kunmap_atomic(vfrom);
831058de DH	339	}
	340	}
	341
5f3ea37c	342	trace_block_bio_bounce(q, *bio_orig);
c43a5082	343
831058de	344	bio->bi_flags \|= (1 << BIO_BOUNCED);
831058de	345
338aa96d	346	if (pool == &page_pool) {
831058de DH	347	bio->bi_end_io = bounce_end_io_write;
	348	if (rw == READ)
	349	bio->bi_end_io = bounce_end_io_read;
	350	} else {
	351	bio->bi_end_io = bounce_end_io_write_isa;
	352	if (rw == READ)
	353	bio->bi_end_io = bounce_end_io_read_isa;
	354	}
	355
	356	bio->bi_private = *bio_orig;
	357	*bio_orig = bio;
	358	}
	359
165125e1	360	void blk_queue_bounce(struct request_queue q, struct bio *bio_orig)
831058de DH	361	{
	362	mempool_t *pool;
	363
bf2de6f5 JA	364	/*
	365	* Data-less bio, nothing to bounce
	366	*/
36144077	367	if (!bio_has_data(*bio_orig))
bf2de6f5 JA	368	return;
bf2de6f5 JA	369
831058de DH	370	/*
	371	* for non-isa bounce case, just check if the bounce pfn is equal
	372	* to or bigger than the highest pfn in the system -- in that case,
	373	* don't waste time iterating over bio segments
	374	*/
	375	if (!(q->bounce_gfp & GFP_DMA)) {
1c4bc3ab	376	if (q->limits.bounce_pfn >= blk_max_pfn)
831058de	377	return;
338aa96d	378	pool = &page_pool;
831058de	379	} else {
338aa96d KO	380	BUG_ON(!mempool_initialized(&isa_page_pool));
338aa96d KO	381	pool = &isa_page_pool;
831058de DH	382	}
831058de DH	383
831058de DH	384	/*
	385	* slow path
	386	*/
a3ad0a9d	387	__blk_queue_bounce(q, bio_orig, pool);
831058de	388	}