[thirdparty/linux.git] / drivers / md / dm-switch.c

/*
 * Copyright (C) 2010-2012 by Dell Inc.  All rights reserved.
 * Copyright (C) 2011-2013 Red Hat, Inc.
 *
 * This file is released under the GPL.
 *
 * dm-switch is a device-mapper target that maps IO to underlying block
 * devices efficiently when there are a large number of fixed-sized
 * address regions but there is no simple pattern to allow for a compact
 * mapping representation such as dm-stripe.
 */

#include <linux/device-mapper.h>

#include <linux/module.h>
#include <linux/init.h>
#include <linux/vmalloc.h>

#define DM_MSG_PREFIX "switch"

/*
 * One region_table_slot_t holds <region_entries_per_slot> region table
 * entries each of which is <region_table_entry_bits> in size.
 */
typedef unsigned long region_table_slot_t;

/*
 * A device with the offset to its start sector.
 */
struct switch_path {
	struct dm_dev *dmdev;
	sector_t start;
};

/*
 * Context block for a dm switch device.
 */
struct switch_ctx {
	struct dm_target *ti;

	unsigned nr_paths;		/* Number of paths in path_list. */

	unsigned region_size;		/* Region size in 512-byte sectors */
	unsigned long nr_regions;	/* Number of regions making up the device */
	signed char region_size_bits;	/* log2 of region_size or -1 */

	unsigned char region_table_entry_bits;	/* Number of bits in one region table entry */
	unsigned char region_entries_per_slot;	/* Number of entries in one region table slot */
	signed char region_entries_per_slot_bits;	/* log2 of region_entries_per_slot or -1 */

	region_table_slot_t *region_table;	/* Region table */

	/*
	 * Array of dm devices to switch between.
	 */
	struct switch_path path_list[0];
};

static struct switch_ctx *alloc_switch_ctx(struct dm_target *ti, unsigned nr_paths,
					   unsigned region_size)
{
	struct switch_ctx *sctx;

	sctx = kzalloc(struct_size(sctx, path_list, nr_paths), GFP_KERNEL);
	if (!sctx)
		return NULL;

	sctx->ti = ti;
	sctx->region_size = region_size;

	ti->private = sctx;

	return sctx;
}

static int alloc_region_table(struct dm_target *ti, unsigned nr_paths)
{
	struct switch_ctx *sctx = ti->private;
	sector_t nr_regions = ti->len;
	sector_t nr_slots;

	if (!(sctx->region_size & (sctx->region_size - 1)))
		sctx->region_size_bits = __ffs(sctx->region_size);
	else
		sctx->region_size_bits = -1;

	sctx->region_table_entry_bits = 1;
	while (sctx->region_table_entry_bits < sizeof(region_table_slot_t) * 8 &&
	       (region_table_slot_t)1 << sctx->region_table_entry_bits < nr_paths)
		sctx->region_table_entry_bits++;

	sctx->region_entries_per_slot = (sizeof(region_table_slot_t) * 8) / sctx->region_table_entry_bits;
	if (!(sctx->region_entries_per_slot & (sctx->region_entries_per_slot - 1)))
		sctx->region_entries_per_slot_bits = __ffs(sctx->region_entries_per_slot);
	else
		sctx->region_entries_per_slot_bits = -1;

	if (sector_div(nr_regions, sctx->region_size))
		nr_regions++;

	if (nr_regions >= ULONG_MAX) {
		ti->error = "Region table too large";
		return -EINVAL;
	}
	sctx->nr_regions = nr_regions;

	nr_slots = nr_regions;
	if (sector_div(nr_slots, sctx->region_entries_per_slot))
		nr_slots++;

	if (nr_slots > ULONG_MAX / sizeof(region_table_slot_t)) {
		ti->error = "Region table too large";
		return -EINVAL;
	}

	sctx->region_table = vmalloc(array_size(nr_slots,
						sizeof(region_table_slot_t)));
	if (!sctx->region_table) {
		ti->error = "Cannot allocate region table";
		return -ENOMEM;
	}

	return 0;
}

static void switch_get_position(struct switch_ctx *sctx, unsigned long region_nr,
				unsigned long *region_index, unsigned *bit)
{
	if (sctx->region_entries_per_slot_bits >= 0) {
		*region_index = region_nr >> sctx->region_entries_per_slot_bits;
		*bit = region_nr & (sctx->region_entries_per_slot - 1);
	} else {
		*region_index = region_nr / sctx->region_entries_per_slot;
		*bit = region_nr % sctx->region_entries_per_slot;
	}

	*bit *= sctx->region_table_entry_bits;
}

static unsigned switch_region_table_read(struct switch_ctx *sctx, unsigned long region_nr)
{
	unsigned long region_index;
	unsigned bit;

	switch_get_position(sctx, region_nr, &region_index, &bit);

	return (READ_ONCE(sctx->region_table[region_index]) >> bit) &
		((1 << sctx->region_table_entry_bits) - 1);
}

/*
 * Find which path to use at given offset.
 */
static unsigned switch_get_path_nr(struct switch_ctx *sctx, sector_t offset)
{
	unsigned path_nr;
	sector_t p;

	p = offset;
	if (sctx->region_size_bits >= 0)
		p >>= sctx->region_size_bits;
	else
		sector_div(p, sctx->region_size);

	path_nr = switch_region_table_read(sctx, p);

	/* This can only happen if the processor uses non-atomic stores. */
	if (unlikely(path_nr >= sctx->nr_paths))
		path_nr = 0;

	return path_nr;
}

static void switch_region_table_write(struct switch_ctx *sctx, unsigned long region_nr,
				      unsigned value)
{
	unsigned long region_index;
	unsigned bit;
	region_table_slot_t pte;

	switch_get_position(sctx, region_nr, &region_index, &bit);

	pte = sctx->region_table[region_index];
	pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit);
	pte |= (region_table_slot_t)value << bit;
	sctx->region_table[region_index] = pte;
}

/*
 * Fill the region table with an initial round robin pattern.
 */
static void initialise_region_table(struct switch_ctx *sctx)
{
	unsigned path_nr = 0;
	unsigned long region_nr;

	for (region_nr = 0; region_nr < sctx->nr_regions; region_nr++) {
		switch_region_table_write(sctx, region_nr, path_nr);
		if (++path_nr >= sctx->nr_paths)
			path_nr = 0;
	}
}

static int parse_path(struct dm_arg_set *as, struct dm_target *ti)
{
	struct switch_ctx *sctx = ti->private;
	unsigned long long start;
	int r;

	r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
			  &sctx->path_list[sctx->nr_paths].dmdev);
	if (r) {
		ti->error = "Device lookup failed";
		return r;
	}

	if (kstrtoull(dm_shift_arg(as), 10, &start) || start != (sector_t)start) {
		ti->error = "Invalid device starting offset";
		dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev);
		return -EINVAL;
	}

	sctx->path_list[sctx->nr_paths].start = start;

	sctx->nr_paths++;

	return 0;
}

/*
 * Destructor: Don't free the dm_target, just the ti->private data (if any).
 */
static void switch_dtr(struct dm_target *ti)
{
	struct switch_ctx *sctx = ti->private;

	while (sctx->nr_paths--)
		dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev);

	vfree(sctx->region_table);
	kfree(sctx);
}

/*
 * Constructor arguments:
 *   <num_paths> <region_size> <num_optional_args> [<optional_args>...]
 *   [<dev_path> <offset>]+
 *
 * Optional args are to allow for future extension: currently this
 * parameter must be 0.
 */
static int switch_ctr(struct dm_target *ti, unsigned argc, char **argv)
{
	static const struct dm_arg _args[] = {
		{1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"},
		{1, UINT_MAX, "Invalid region size"},
		{0, 0, "Invalid number of optional args"},
	};

	struct switch_ctx *sctx;
	struct dm_arg_set as;
	unsigned nr_paths, region_size, nr_optional_args;
	int r;

	as.argc = argc;
	as.argv = argv;

	r = dm_read_arg(_args, &as, &nr_paths, &ti->error);
	if (r)
		return -EINVAL;

	r = dm_read_arg(_args + 1, &as, &region_size, &ti->error);
	if (r)
		return r;

	r = dm_read_arg_group(_args + 2, &as, &nr_optional_args, &ti->error);
	if (r)
		return r;
	/* parse optional arguments here, if we add any */

	if (as.argc != nr_paths * 2) {
		ti->error = "Incorrect number of path arguments";
		return -EINVAL;
	}

	sctx = alloc_switch_ctx(ti, nr_paths, region_size);
	if (!sctx) {
		ti->error = "Cannot allocate redirection context";
		return -ENOMEM;
	}

	r = dm_set_target_max_io_len(ti, region_size);
	if (r)
		goto error;

	while (as.argc) {
		r = parse_path(&as, ti);
		if (r)
			goto error;
	}

	r = alloc_region_table(ti, nr_paths);
	if (r)
		goto error;

	initialise_region_table(sctx);

	/* For UNMAP, sending the request down any path is sufficient */
	ti->num_discard_bios = 1;

	return 0;

error:
	switch_dtr(ti);

	return r;
}

static int switch_map(struct dm_target *ti, struct bio *bio)
{
	struct switch_ctx *sctx = ti->private;
	sector_t offset = dm_target_offset(ti, bio->bi_iter.bi_sector);
	unsigned path_nr = switch_get_path_nr(sctx, offset);

	bio_set_dev(bio, sctx->path_list[path_nr].dmdev->bdev);
	bio->bi_iter.bi_sector = sctx->path_list[path_nr].start + offset;

	return DM_MAPIO_REMAPPED;
}

/*
 * We need to parse hex numbers in the message as quickly as possible.
 *
 * This table-based hex parser improves performance.
 * It improves a time to load 1000000 entries compared to the condition-based
 * parser.
 *		table-based parser	condition-based parser
 * PA-RISC	0.29s			0.31s
 * Opteron	0.0495s			0.0498s
 */
static const unsigned char hex_table[256] = {
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255
};

static __always_inline unsigned long parse_hex(const char **string)
{
	unsigned char d;
	unsigned long r = 0;

	while ((d = hex_table[(unsigned char)**string]) < 16) {
		r = (r << 4) | d;
		(*string)++;
	}

	return r;
}

static int process_set_region_mappings(struct switch_ctx *sctx,
				       unsigned argc, char **argv)
{
	unsigned i;
	unsigned long region_index = 0;

	for (i = 1; i < argc; i++) {
		unsigned long path_nr;
		const char *string = argv[i];

		if ((*string & 0xdf) == 'R') {
			unsigned long cycle_length, num_write;

			string++;
			if (unlikely(*string == ',')) {
				DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
				return -EINVAL;
			}
			cycle_length = parse_hex(&string);
			if (unlikely(*string != ',')) {
				DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
				return -EINVAL;
			}
			string++;
			if (unlikely(!*string)) {
				DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
				return -EINVAL;
			}
			num_write = parse_hex(&string);
			if (unlikely(*string)) {
				DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
				return -EINVAL;
			}

			if (unlikely(!cycle_length) || unlikely(cycle_length - 1 > region_index)) {
				DMWARN("invalid set_region_mappings cycle length: %lu > %lu",
				       cycle_length - 1, region_index);
				return -EINVAL;
			}
			if (unlikely(region_index + num_write < region_index) ||
			    unlikely(region_index + num_write >= sctx->nr_regions)) {
				DMWARN("invalid set_region_mappings region number: %lu + %lu >= %lu",
				       region_index, num_write, sctx->nr_regions);
				return -EINVAL;
			}

			while (num_write--) {
				region_index++;
				path_nr = switch_region_table_read(sctx, region_index - cycle_length);
				switch_region_table_write(sctx, region_index, path_nr);
			}

			continue;
		}

		if (*string == ':')
			region_index++;
		else {
			region_index = parse_hex(&string);
			if (unlikely(*string != ':')) {
				DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
				return -EINVAL;
			}
		}

		string++;
		if (unlikely(!*string)) {
			DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
			return -EINVAL;
		}

		path_nr = parse_hex(&string);
		if (unlikely(*string)) {
			DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
			return -EINVAL;
		}
		if (unlikely(region_index >= sctx->nr_regions)) {
			DMWARN("invalid set_region_mappings region number: %lu >= %lu", region_index, sctx->nr_regions);
			return -EINVAL;
		}
		if (unlikely(path_nr >= sctx->nr_paths)) {
			DMWARN("invalid set_region_mappings device: %lu >= %u", path_nr, sctx->nr_paths);
			return -EINVAL;
		}

		switch_region_table_write(sctx, region_index, path_nr);
	}

	return 0;
}

/*
 * Messages are processed one-at-a-time.
 *
 * Only set_region_mappings is supported.
 */
static int switch_message(struct dm_target *ti, unsigned argc, char **argv,
			  char *result, unsigned maxlen)
{
	static DEFINE_MUTEX(message_mutex);

	struct switch_ctx *sctx = ti->private;
	int r = -EINVAL;

	mutex_lock(&message_mutex);

	if (!strcasecmp(argv[0], "set_region_mappings"))
		r = process_set_region_mappings(sctx, argc, argv);
	else
		DMWARN("Unrecognised message received.");

	mutex_unlock(&message_mutex);

	return r;
}

static void switch_status(struct dm_target *ti, status_type_t type,
			  unsigned status_flags, char *result, unsigned maxlen)
{
	struct switch_ctx *sctx = ti->private;
	unsigned sz = 0;
	int path_nr;

	switch (type) {
	case STATUSTYPE_INFO:
		result[0] = '\0';
		break;

	case STATUSTYPE_TABLE:
		DMEMIT("%u %u 0", sctx->nr_paths, sctx->region_size);
		for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++)
			DMEMIT(" %s %llu", sctx->path_list[path_nr].dmdev->name,
			       (unsigned long long)sctx->path_list[path_nr].start);
		break;
	}
}

/*
 * Switch ioctl:
 *
 * Passthrough all ioctls to the path for sector 0
 */
static int switch_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
{
	struct switch_ctx *sctx = ti->private;
	unsigned path_nr;

	path_nr = switch_get_path_nr(sctx, 0);

	*bdev = sctx->path_list[path_nr].dmdev->bdev;

	/*
	 * Only pass ioctls through if the device sizes match exactly.
	 */
	if (ti->len + sctx->path_list[path_nr].start !=
	    i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
		return 1;
	return 0;
}

static int switch_iterate_devices(struct dm_target *ti,
				  iterate_devices_callout_fn fn, void *data)
{
	struct switch_ctx *sctx = ti->private;
	int path_nr;
	int r;

	for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++) {
		r = fn(ti, sctx->path_list[path_nr].dmdev,
			 sctx->path_list[path_nr].start, ti->len, data);
		if (r)
			return r;
	}

	return 0;
}

static struct target_type switch_target = {
	.name = "switch",
	.version = {1, 1, 0},
	.module = THIS_MODULE,
	.ctr = switch_ctr,
	.dtr = switch_dtr,
	.map = switch_map,
	.message = switch_message,
	.status = switch_status,
	.prepare_ioctl = switch_prepare_ioctl,
	.iterate_devices = switch_iterate_devices,
};

static int __init dm_switch_init(void)
{
	int r;

	r = dm_register_target(&switch_target);
	if (r < 0)
		DMERR("dm_register_target() failed %d", r);

	return r;
}

static void __exit dm_switch_exit(void)
{
	dm_unregister_target(&switch_target);
}

module_init(dm_switch_init);
module_exit(dm_switch_exit);

MODULE_DESCRIPTION(DM_NAME " dynamic path switching target");
MODULE_AUTHOR("Kevin D. O'Kelley <Kevin_OKelley@dell.com>");
MODULE_AUTHOR("Narendran Ganapathy <Narendran_Ganapathy@dell.com>");
MODULE_AUTHOR("Jim Ramsay <Jim_Ramsay@dell.com>");
MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
MODULE_LICENSE("GPL");
Commit	Line	Data
9d0eb0ab JR	1	/*
	2	* Copyright (C) 2010-2012 by Dell Inc. All rights reserved.
	3	* Copyright (C) 2011-2013 Red Hat, Inc.
	4	*
	5	* This file is released under the GPL.
	6	*
	7	* dm-switch is a device-mapper target that maps IO to underlying block
	8	* devices efficiently when there are a large number of fixed-sized
	9	* address regions but there is no simple pattern to allow for a compact
	10	* mapping representation such as dm-stripe.
	11	*/
	12
	13	#include <linux/device-mapper.h>
	14
	15	#include <linux/module.h>
	16	#include <linux/init.h>
	17	#include <linux/vmalloc.h>
	18
	19	#define DM_MSG_PREFIX "switch"
	20
	21	/*
	22	* One region_table_slot_t holds <region_entries_per_slot> region table
	23	* entries each of which is <region_table_entry_bits> in size.
	24	*/
	25	typedef unsigned long region_table_slot_t;
	26
	27	/*
	28	* A device with the offset to its start sector.
	29	*/
	30	struct switch_path {
	31	struct dm_dev *dmdev;
	32	sector_t start;
	33	};
	34
	35	/*
	36	* Context block for a dm switch device.
	37	*/
	38	struct switch_ctx {
	39	struct dm_target *ti;
	40
	41	unsigned nr_paths; /* Number of paths in path_list. */
	42
	43	unsigned region_size; /* Region size in 512-byte sectors */
	44	unsigned long nr_regions; /* Number of regions making up the device */
	45	signed char region_size_bits; /* log2 of region_size or -1 */
	46
	47	unsigned char region_table_entry_bits; /* Number of bits in one region table entry */
	48	unsigned char region_entries_per_slot; /* Number of entries in one region table slot */
	49	signed char region_entries_per_slot_bits; /* log2 of region_entries_per_slot or -1 */
	50
	51	region_table_slot_t region_table; / Region table */
	52
	53	/*
	54	* Array of dm devices to switch between.
	55	*/
	56	struct switch_path path_list[0];
	57	};
	58
	59	static struct switch_ctx alloc_switch_ctx(struct dm_target ti, unsigned nr_paths,
	60	unsigned region_size)
	61	{
	62	struct switch_ctx *sctx;
	63
d2832376	64	sctx = kzalloc(struct_size(sctx, path_list, nr_paths), GFP_KERNEL);
9d0eb0ab JR	65	if (!sctx)
	66	return NULL;
	67
	68	sctx->ti = ti;
	69	sctx->region_size = region_size;
	70
	71	ti->private = sctx;
	72
	73	return sctx;
	74	}
	75
	76	static int alloc_region_table(struct dm_target *ti, unsigned nr_paths)
	77	{
	78	struct switch_ctx *sctx = ti->private;
	79	sector_t nr_regions = ti->len;
	80	sector_t nr_slots;
	81
	82	if (!(sctx->region_size & (sctx->region_size - 1)))
	83	sctx->region_size_bits = __ffs(sctx->region_size);
	84	else
	85	sctx->region_size_bits = -1;
	86
	87	sctx->region_table_entry_bits = 1;
	88	while (sctx->region_table_entry_bits < sizeof(region_table_slot_t) * 8 &&
	89	(region_table_slot_t)1 << sctx->region_table_entry_bits < nr_paths)
	90	sctx->region_table_entry_bits++;
	91
	92	sctx->region_entries_per_slot = (sizeof(region_table_slot_t) * 8) / sctx->region_table_entry_bits;
	93	if (!(sctx->region_entries_per_slot & (sctx->region_entries_per_slot - 1)))
	94	sctx->region_entries_per_slot_bits = __ffs(sctx->region_entries_per_slot);
	95	else
	96	sctx->region_entries_per_slot_bits = -1;
	97
	98	if (sector_div(nr_regions, sctx->region_size))
	99	nr_regions++;
	100
aad9ae45	101	if (nr_regions >= ULONG_MAX) {
9d0eb0ab JR	102	ti->error = "Region table too large";
	103	return -EINVAL;
	104	}
aad9ae45	105	sctx->nr_regions = nr_regions;
9d0eb0ab JR	106
	107	nr_slots = nr_regions;
	108	if (sector_div(nr_slots, sctx->region_entries_per_slot))
	109	nr_slots++;
	110
	111	if (nr_slots > ULONG_MAX / sizeof(region_table_slot_t)) {
	112	ti->error = "Region table too large";
	113	return -EINVAL;
	114	}
	115
42bc47b3 KC	116	sctx->region_table = vmalloc(array_size(nr_slots,
42bc47b3 KC	117	sizeof(region_table_slot_t)));
9d0eb0ab JR	118	if (!sctx->region_table) {
	119	ti->error = "Cannot allocate region table";
	120	return -ENOMEM;
	121	}
	122
	123	return 0;
	124	}
	125
	126	static void switch_get_position(struct switch_ctx *sctx, unsigned long region_nr,
	127	unsigned long region_index, unsigned bit)
	128	{
	129	if (sctx->region_entries_per_slot_bits >= 0) {
	130	*region_index = region_nr >> sctx->region_entries_per_slot_bits;
	131	*bit = region_nr & (sctx->region_entries_per_slot - 1);
	132	} else {
	133	*region_index = region_nr / sctx->region_entries_per_slot;
	134	*bit = region_nr % sctx->region_entries_per_slot;
	135	}
	136
	137	bit = sctx->region_table_entry_bits;
	138	}
	139
99eb1908 MP	140	static unsigned switch_region_table_read(struct switch_ctx *sctx, unsigned long region_nr)
	141	{
	142	unsigned long region_index;
	143	unsigned bit;
	144
	145	switch_get_position(sctx, region_nr, &region_index, &bit);
	146
6aa7de05	147	return (READ_ONCE(sctx->region_table[region_index]) >> bit) &
99eb1908 MP	148	((1 << sctx->region_table_entry_bits) - 1);
	149	}
	150
9d0eb0ab JR	151	/*
	152	* Find which path to use at given offset.
	153	*/
	154	static unsigned switch_get_path_nr(struct switch_ctx *sctx, sector_t offset)
	155	{
99eb1908	156	unsigned path_nr;
9d0eb0ab JR	157	sector_t p;
	158
	159	p = offset;
	160	if (sctx->region_size_bits >= 0)
	161	p >>= sctx->region_size_bits;
	162	else
	163	sector_div(p, sctx->region_size);
	164
99eb1908	165	path_nr = switch_region_table_read(sctx, p);
9d0eb0ab JR	166
	167	/* This can only happen if the processor uses non-atomic stores. */
	168	if (unlikely(path_nr >= sctx->nr_paths))
	169	path_nr = 0;
	170
	171	return path_nr;
	172	}
	173
	174	static void switch_region_table_write(struct switch_ctx *sctx, unsigned long region_nr,
	175	unsigned value)
	176	{
	177	unsigned long region_index;
	178	unsigned bit;
	179	region_table_slot_t pte;
	180
	181	switch_get_position(sctx, region_nr, &region_index, &bit);
	182
	183	pte = sctx->region_table[region_index];
	184	pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit);
	185	pte \|= (region_table_slot_t)value << bit;
	186	sctx->region_table[region_index] = pte;
	187	}
	188
	189	/*
	190	* Fill the region table with an initial round robin pattern.
	191	*/
	192	static void initialise_region_table(struct switch_ctx *sctx)
	193	{
	194	unsigned path_nr = 0;
	195	unsigned long region_nr;
	196
	197	for (region_nr = 0; region_nr < sctx->nr_regions; region_nr++) {
	198	switch_region_table_write(sctx, region_nr, path_nr);
	199	if (++path_nr >= sctx->nr_paths)
	200	path_nr = 0;
	201	}
	202	}
	203
	204	static int parse_path(struct dm_arg_set as, struct dm_target ti)
	205	{
	206	struct switch_ctx *sctx = ti->private;
	207	unsigned long long start;
	208	int r;
	209
	210	r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
	211	&sctx->path_list[sctx->nr_paths].dmdev);
	212	if (r) {
	213	ti->error = "Device lookup failed";
	214	return r;
	215	}
	216
	217	if (kstrtoull(dm_shift_arg(as), 10, &start) \|\| start != (sector_t)start) {
	218	ti->error = "Invalid device starting offset";
	219	dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev);
	220	return -EINVAL;
	221	}
	222
	223	sctx->path_list[sctx->nr_paths].start = start;
	224
	225	sctx->nr_paths++;
	226
	227	return 0;
	228	}
	229
230	/*
231	* Destructor: Don't free the dm_target, just the ti->private data (if any).
232	*/
233	static void switch_dtr(struct dm_target *ti)
234	{
235	struct switch_ctx *sctx = ti->private;
236
237	while (sctx->nr_paths--)
238	dm_put_device(ti, sctx->path_list[sctx->nr_paths].dmdev);
239
240	vfree(sctx->region_table);
241	kfree(sctx);
242	}
243
244	/*
245	* Constructor arguments:
246	* <num_paths> <region_size> <num_optional_args> [<optional_args>...]
247	* [<dev_path> <offset>]+
248	*
249	* Optional args are to allow for future extension: currently this
250	* parameter must be 0.
251	*/
252	static int switch_ctr(struct dm_target ti, unsigned argc, char *argv)
253	{
5916a22b	254	static const struct dm_arg _args[] = {
9d0eb0ab JR	255	{1, (KMALLOC_MAX_SIZE - sizeof(struct switch_ctx)) / sizeof(struct switch_path), "Invalid number of paths"},
	256	{1, UINT_MAX, "Invalid region size"},
	257	{0, 0, "Invalid number of optional args"},
	258	};
	259
	260	struct switch_ctx *sctx;
	261	struct dm_arg_set as;
	262	unsigned nr_paths, region_size, nr_optional_args;
	263	int r;
	264
	265	as.argc = argc;
	266	as.argv = argv;
	267
	268	r = dm_read_arg(_args, &as, &nr_paths, &ti->error);
	269	if (r)
	270	return -EINVAL;
	271
	272	r = dm_read_arg(_args + 1, &as, &region_size, &ti->error);
	273	if (r)
	274	return r;
	275
	276	r = dm_read_arg_group(_args + 2, &as, &nr_optional_args, &ti->error);
	277	if (r)
	278	return r;
	279	/* parse optional arguments here, if we add any */
	280
	281	if (as.argc != nr_paths * 2) {
	282	ti->error = "Incorrect number of path arguments";
	283	return -EINVAL;
	284	}
	285
	286	sctx = alloc_switch_ctx(ti, nr_paths, region_size);
	287	if (!sctx) {
	288	ti->error = "Cannot allocate redirection context";
	289	return -ENOMEM;
	290	}
	291
	292	r = dm_set_target_max_io_len(ti, region_size);
	293	if (r)
	294	goto error;
	295
	296	while (as.argc) {
	297	r = parse_path(&as, ti);
	298	if (r)
	299	goto error;
	300	}
	301
	302	r = alloc_region_table(ti, nr_paths);
	303	if (r)
	304	goto error;
	305
	306	initialise_region_table(sctx);
	307
	308	/* For UNMAP, sending the request down any path is sufficient */
	309	ti->num_discard_bios = 1;
	310
	311	return 0;
	312
	313	error:
	314	switch_dtr(ti);
	315
	316	return r;
	317	}
	318
319	static int switch_map(struct dm_target ti, struct bio bio)
320	{
321	struct switch_ctx *sctx = ti->private;
4f024f37	322	sector_t offset = dm_target_offset(ti, bio->bi_iter.bi_sector);
9d0eb0ab JR	323	unsigned path_nr = switch_get_path_nr(sctx, offset);
9d0eb0ab JR	324
74d46992	325	bio_set_dev(bio, sctx->path_list[path_nr].dmdev->bdev);
4f024f37	326	bio->bi_iter.bi_sector = sctx->path_list[path_nr].start + offset;
9d0eb0ab JR	327
	328	return DM_MAPIO_REMAPPED;
	329	}
	330
	331	/*
	332	* We need to parse hex numbers in the message as quickly as possible.
	333	*
	334	* This table-based hex parser improves performance.
	335	* It improves a time to load 1000000 entries compared to the condition-based
	336	* parser.
	337	* table-based parser condition-based parser
	338	* PA-RISC 0.29s 0.31s
	339	* Opteron 0.0495s 0.0498s
	340	*/
	341	static const unsigned char hex_table[256] = {
	342	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	343	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	344	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	345	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
	346	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	347	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	348	255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	349	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	350	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	351	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	352	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	353	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	354	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	355	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	356	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
	357	255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255
	358	};
	359
	360	static __always_inline unsigned long parse_hex(const char **string)
	361	{
	362	unsigned char d;
	363	unsigned long r = 0;
	364
	365	while ((d = hex_table[(unsigned char)**string]) < 16) {
	366	r = (r << 4) \| d;
	367	(*string)++;
	368	}
	369
	370	return r;
	371	}
	372
	373	static int process_set_region_mappings(struct switch_ctx *sctx,
56b1ebf2	374	unsigned argc, char **argv)
9d0eb0ab JR	375	{
	376	unsigned i;
	377	unsigned long region_index = 0;
	378
	379	for (i = 1; i < argc; i++) {
	380	unsigned long path_nr;
	381	const char *string = argv[i];
	382
56b1ebf2 MP	383	if ((*string & 0xdf) == 'R') {
	384	unsigned long cycle_length, num_write;
	385
	386	string++;
	387	if (unlikely(*string == ',')) {
	388	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	389	return -EINVAL;
	390	}
	391	cycle_length = parse_hex(&string);
	392	if (unlikely(*string != ',')) {
	393	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	394	return -EINVAL;
	395	}
	396	string++;
	397	if (unlikely(!*string)) {
	398	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	399	return -EINVAL;
	400	}
	401	num_write = parse_hex(&string);
	402	if (unlikely(*string)) {
	403	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	404	return -EINVAL;
	405	}
	406
	407	if (unlikely(!cycle_length) \|\| unlikely(cycle_length - 1 > region_index)) {
	408	DMWARN("invalid set_region_mappings cycle length: %lu > %lu",
	409	cycle_length - 1, region_index);
	410	return -EINVAL;
	411	}
	412	if (unlikely(region_index + num_write < region_index) \|\|
	413	unlikely(region_index + num_write >= sctx->nr_regions)) {
	414	DMWARN("invalid set_region_mappings region number: %lu + %lu >= %lu",
	415	region_index, num_write, sctx->nr_regions);
	416	return -EINVAL;
	417	}
	418
	419	while (num_write--) {
	420	region_index++;
	421	path_nr = switch_region_table_read(sctx, region_index - cycle_length);
	422	switch_region_table_write(sctx, region_index, path_nr);
	423	}
	424
	425	continue;
	426	}
	427
9d0eb0ab JR	428	if (*string == ':')
	429	region_index++;
	430	else {
	431	region_index = parse_hex(&string);
	432	if (unlikely(*string != ':')) {
	433	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	434	return -EINVAL;
	435	}
	436	}
	437
	438	string++;
	439	if (unlikely(!*string)) {
	440	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	441	return -EINVAL;
	442	}
	443
	444	path_nr = parse_hex(&string);
	445	if (unlikely(*string)) {
	446	DMWARN("invalid set_region_mappings argument: '%s'", argv[i]);
	447	return -EINVAL;
	448	}
	449	if (unlikely(region_index >= sctx->nr_regions)) {
	450	DMWARN("invalid set_region_mappings region number: %lu >= %lu", region_index, sctx->nr_regions);
	451	return -EINVAL;
	452	}
	453	if (unlikely(path_nr >= sctx->nr_paths)) {
	454	DMWARN("invalid set_region_mappings device: %lu >= %u", path_nr, sctx->nr_paths);
	455	return -EINVAL;
	456	}
	457
	458	switch_region_table_write(sctx, region_index, path_nr);
	459	}
	460
	461	return 0;
	462	}
	463
	464	/*
	465	* Messages are processed one-at-a-time.
	466	*
	467	* Only set_region_mappings is supported.
	468	*/
1eb5fa84 MS	469	static int switch_message(struct dm_target ti, unsigned argc, char *argv,
1eb5fa84 MS	470	char *result, unsigned maxlen)
9d0eb0ab JR	471	{
	472	static DEFINE_MUTEX(message_mutex);
	473
	474	struct switch_ctx *sctx = ti->private;
	475	int r = -EINVAL;
	476
	477	mutex_lock(&message_mutex);
	478
	479	if (!strcasecmp(argv[0], "set_region_mappings"))
	480	r = process_set_region_mappings(sctx, argc, argv);
	481	else
	482	DMWARN("Unrecognised message received.");
	483
	484	mutex_unlock(&message_mutex);
	485
	486	return r;
	487	}
	488
	489	static void switch_status(struct dm_target *ti, status_type_t type,
	490	unsigned status_flags, char *result, unsigned maxlen)
	491	{
	492	struct switch_ctx *sctx = ti->private;
	493	unsigned sz = 0;
	494	int path_nr;
	495
	496	switch (type) {
	497	case STATUSTYPE_INFO:
	498	result[0] = '\0';
	499	break;
	500
	501	case STATUSTYPE_TABLE:
	502	DMEMIT("%u %u 0", sctx->nr_paths, sctx->region_size);
	503	for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++)
	504	DMEMIT(" %s %llu", sctx->path_list[path_nr].dmdev->name,
	505	(unsigned long long)sctx->path_list[path_nr].start);
	506	break;
	507	}
	508	}
	509
	510	/*
	511	* Switch ioctl:
	512	*
	513	* Passthrough all ioctls to the path for sector 0
	514	*/
5bd5e8d8	515	static int switch_prepare_ioctl(struct dm_target ti, struct block_device *bdev)
9d0eb0ab JR	516	{
9d0eb0ab JR	517	struct switch_ctx *sctx = ti->private;
9d0eb0ab	518	unsigned path_nr;
9d0eb0ab JR	519
	520	path_nr = switch_get_path_nr(sctx, 0);
	521
e56f81e0	522	*bdev = sctx->path_list[path_nr].dmdev->bdev;
9d0eb0ab JR	523
	524	/*
	525	* Only pass ioctls through if the device sizes match exactly.
	526	*/
e56f81e0 CH	527	if (ti->len + sctx->path_list[path_nr].start !=
	528	i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
	529	return 1;
	530	return 0;
9d0eb0ab JR	531	}
	532
	533	static int switch_iterate_devices(struct dm_target *ti,
	534	iterate_devices_callout_fn fn, void *data)
	535	{
	536	struct switch_ctx *sctx = ti->private;
	537	int path_nr;
	538	int r;
	539
	540	for (path_nr = 0; path_nr < sctx->nr_paths; path_nr++) {
	541	r = fn(ti, sctx->path_list[path_nr].dmdev,
	542	sctx->path_list[path_nr].start, ti->len, data);
	543	if (r)
	544	return r;
	545	}
	546
	547	return 0;
	548	}
	549
	550	static struct target_type switch_target = {
	551	.name = "switch",
56b1ebf2	552	.version = {1, 1, 0},
9d0eb0ab JR	553	.module = THIS_MODULE,
	554	.ctr = switch_ctr,
	555	.dtr = switch_dtr,
	556	.map = switch_map,
	557	.message = switch_message,
	558	.status = switch_status,
e56f81e0	559	.prepare_ioctl = switch_prepare_ioctl,
9d0eb0ab JR	560	.iterate_devices = switch_iterate_devices,
	561	};
	562
	563	static int __init dm_switch_init(void)
	564	{
	565	int r;
	566
	567	r = dm_register_target(&switch_target);
	568	if (r < 0)
	569	DMERR("dm_register_target() failed %d", r);
	570
	571	return r;
	572	}
	573
	574	static void __exit dm_switch_exit(void)
	575	{
	576	dm_unregister_target(&switch_target);
	577	}
	578
	579	module_init(dm_switch_init);
	580	module_exit(dm_switch_exit);
	581
	582	MODULE_DESCRIPTION(DM_NAME " dynamic path switching target");
	583	MODULE_AUTHOR("Kevin D. O'Kelley <Kevin_OKelley@dell.com>");
	584	MODULE_AUTHOR("Narendran Ganapathy <Narendran_Ganapathy@dell.com>");
	585	MODULE_AUTHOR("Jim Ramsay <Jim_Ramsay@dell.com>");
	586	MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
	587	MODULE_LICENSE("GPL");