Grow: be extra careful about races when freezing an array

[thirdparty/mdadm.git] / Grow.c

/*
 * mdadm - manage Linux "md" devices aka RAID arrays.
 *
 * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
 *
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 *    Author: Neil Brown
 *    Email: <neilb@suse.de>
 */
#include	"mdadm.h"
#include	"dlink.h"
#include	<sys/mman.h>

#if ! defined(__BIG_ENDIAN) && ! defined(__LITTLE_ENDIAN)
#error no endian defined
#endif
#include	"md_u.h"
#include	"md_p.h"

#ifndef offsetof
#define offsetof(t,f) ((size_t)&(((t*)0)->f))
#endif

int Grow_Add_device(char *devname, int fd, char *newdev)
{
	/* Add a device to an active array.
	 * Currently, just extend a linear array.
	 * This requires writing a new superblock on the
	 * new device, calling the kernel to add the device,
	 * and if that succeeds, update the superblock on
	 * all other devices.
	 * This means that we need to *find* all other devices.
	 */
	struct mdinfo info;

	struct stat stb;
	int nfd, fd2;
	int d, nd;
	struct supertype *st = NULL;
	char *subarray = NULL;

	if (ioctl(fd, GET_ARRAY_INFO, &info.array) < 0) {
		fprintf(stderr, Name ": cannot get array info for %s\n", devname);
		return 1;
	}

	if (info.array.level != -1) {
		fprintf(stderr, Name ": can only add devices to linear arrays\n");
		return 1;
	}

	st = super_by_fd(fd, &subarray);
	if (!st) {
		fprintf(stderr, Name ": cannot handle arrays with superblock version %d\n", info.array.major_version);
		return 1;
	}

	if (subarray) {
		fprintf(stderr, Name ": Cannot grow linear sub-arrays yet\n");
		free(subarray);
		free(st);
	}

	nfd = open(newdev, O_RDWR|O_EXCL|O_DIRECT);
	if (nfd < 0) {
		fprintf(stderr, Name ": cannot open %s\n", newdev);
		free(st);
		return 1;
	}
	fstat(nfd, &stb);
	if ((stb.st_mode & S_IFMT) != S_IFBLK) {
		fprintf(stderr, Name ": %s is not a block device!\n", newdev);
		close(nfd);
		free(st);
		return 1;
	}
	/* now check out all the devices and make sure we can read the superblock */
	for (d=0 ; d < info.array.raid_disks ; d++) {
		mdu_disk_info_t disk;
		char *dv;

		st->ss->free_super(st);

		disk.number = d;
		if (ioctl(fd, GET_DISK_INFO, &disk) < 0) {
			fprintf(stderr, Name ": cannot get device detail for device %d\n",
				d);
			close(nfd);
			free(st);
			return 1;
		}
		dv = map_dev(disk.major, disk.minor, 1);
		if (!dv) {
			fprintf(stderr, Name ": cannot find device file for device %d\n",
				d);
			close(nfd);
			free(st);
			return 1;
		}
		fd2 = dev_open(dv, O_RDWR);
		if (!fd2) {
			fprintf(stderr, Name ": cannot open device file %s\n", dv);
			close(nfd);
			free(st);
			return 1;
		}

		if (st->ss->load_super(st, fd2, NULL)) {
			fprintf(stderr, Name ": cannot find super block on %s\n", dv);
			close(nfd);
			close(fd2);
			free(st);
			return 1;
		}
		close(fd2);
	}
	/* Ok, looks good. Lets update the superblock and write it out to
	 * newdev.
	 */

	info.disk.number = d;
	info.disk.major = major(stb.st_rdev);
	info.disk.minor = minor(stb.st_rdev);
	info.disk.raid_disk = d;
	info.disk.state = (1 << MD_DISK_SYNC) | (1 << MD_DISK_ACTIVE);
	st->ss->update_super(st, &info, "linear-grow-new", newdev,
			     0, 0, NULL);

	if (st->ss->store_super(st, nfd)) {
		fprintf(stderr, Name ": Cannot store new superblock on %s\n",
			newdev);
		close(nfd);
		return 1;
	}
	close(nfd);

	if (ioctl(fd, ADD_NEW_DISK, &info.disk) != 0) {
		fprintf(stderr, Name ": Cannot add new disk to this array\n");
		return 1;
	}
	/* Well, that seems to have worked.
	 * Now go through and update all superblocks
	 */

	if (ioctl(fd, GET_ARRAY_INFO, &info.array) < 0) {
		fprintf(stderr, Name ": cannot get array info for %s\n", devname);
		return 1;
	}

	nd = d;
	for (d=0 ; d < info.array.raid_disks ; d++) {
		mdu_disk_info_t disk;
		char *dv;

		disk.number = d;
		if (ioctl(fd, GET_DISK_INFO, &disk) < 0) {
			fprintf(stderr, Name ": cannot get device detail for device %d\n",
				d);
			return 1;
		}
		dv = map_dev(disk.major, disk.minor, 1);
		if (!dv) {
			fprintf(stderr, Name ": cannot find device file for device %d\n",
				d);
			return 1;
		}
		fd2 = dev_open(dv, O_RDWR);
		if (fd2 < 0) {
			fprintf(stderr, Name ": cannot open device file %s\n", dv);
			return 1;
		}
		if (st->ss->load_super(st, fd2, NULL)) {
			fprintf(stderr, Name ": cannot find super block on %s\n", dv);
			close(fd);
			return 1;
		}
		info.array.raid_disks = nd+1;
		info.array.nr_disks = nd+1;
		info.array.active_disks = nd+1;
		info.array.working_disks = nd+1;

		st->ss->update_super(st, &info, "linear-grow-update", dv,
				     0, 0, NULL);

		if (st->ss->store_super(st, fd2)) {
			fprintf(stderr, Name ": Cannot store new superblock on %s\n", dv);
			close(fd2);
			return 1;
		}
		close(fd2);
	}

	return 0;
}

int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int write_behind, int force)
{
	/*
	 * First check that array doesn't have a bitmap
	 * Then create the bitmap
	 * Then add it
	 *
	 * For internal bitmaps, we need to check the version,
	 * find all the active devices, and write the bitmap block
	 * to all devices
	 */
	mdu_bitmap_file_t bmf;
	mdu_array_info_t array;
	struct supertype *st;
	char *subarray = NULL;
	int major = BITMAP_MAJOR_HI;
	int vers = md_get_version(fd);
	unsigned long long bitmapsize, array_size;

	if (vers < 9003) {
		major = BITMAP_MAJOR_HOSTENDIAN;
		fprintf(stderr, Name ": Warning - bitmaps created on this kernel"
			" are not portable\n"
			"  between different architectures.  Consider upgrading"
			" the Linux kernel.\n");
	}

	if (ioctl(fd, GET_BITMAP_FILE, &bmf) != 0) {
		if (errno == ENOMEM)
			fprintf(stderr, Name ": Memory allocation failure.\n");
		else
			fprintf(stderr, Name ": bitmaps not supported by this kernel.\n");
		return 1;
	}
	if (bmf.pathname[0]) {
		if (strcmp(file,"none")==0) {
			if (ioctl(fd, SET_BITMAP_FILE, -1)!= 0) {
				fprintf(stderr, Name ": failed to remove bitmap %s\n",
					bmf.pathname);
				return 1;
			}
			return 0;
		}
		fprintf(stderr, Name ": %s already has a bitmap (%s)\n",
			devname, bmf.pathname);
		return 1;
	}
	if (ioctl(fd, GET_ARRAY_INFO, &array) != 0) {
		fprintf(stderr, Name ": cannot get array status for %s\n", devname);
		return 1;
	}
	if (array.state & (1<<MD_SB_BITMAP_PRESENT)) {
		if (strcmp(file, "none")==0) {
			array.state &= ~(1<<MD_SB_BITMAP_PRESENT);
			if (ioctl(fd, SET_ARRAY_INFO, &array)!= 0) {
				fprintf(stderr, Name ": failed to remove internal bitmap.\n");
				return 1;
			}
			return 0;
		}
		fprintf(stderr, Name ": Internal bitmap already present on %s\n",
			devname);
		return 1;
	}

	if (strcmp(file, "none") == 0) {
		fprintf(stderr, Name ": no bitmap found on %s\n", devname);
		return 1;
	}
	if (array.level <= 0) {
		fprintf(stderr, Name ": Bitmaps not meaningful with level %s\n",
			map_num(pers, array.level)?:"of this array");
		return 1;
	}
	bitmapsize = array.size;
	bitmapsize <<= 1;
	if (get_dev_size(fd, NULL, &array_size) &&
	    array_size > (0x7fffffffULL<<9)) {
		/* Array is big enough that we cannot trust array.size
		 * try other approaches
		 */
		bitmapsize = get_component_size(fd);
	}
	if (bitmapsize == 0) {
		fprintf(stderr, Name ": Cannot reliably determine size of array to create bitmap - sorry.\n");
		return 1;
	}

	if (array.level == 10) {
		int ncopies = (array.layout&255)*((array.layout>>8)&255);
		bitmapsize = bitmapsize * array.raid_disks / ncopies;
	}

	st = super_by_fd(fd, &subarray);
	if (!st) {
		fprintf(stderr, Name ": Cannot understand version %d.%d\n",
			array.major_version, array.minor_version);
		return 1;
	}
	if (subarray) {
		fprintf(stderr, Name ": Cannot add bitmaps to sub-arrays yet\n");
		free(subarray);
		free(st);
		return 1;
	}
	if (strcmp(file, "internal") == 0) {
		int d;
		if (st->ss->add_internal_bitmap == NULL) {
			fprintf(stderr, Name ": Internal bitmaps not supported "
				"with %s metadata\n", st->ss->name);
			return 1;
		}
		for (d=0; d< st->max_devs; d++) {
			mdu_disk_info_t disk;
			char *dv;
			disk.number = d;
			if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
				continue;
			if (disk.major == 0 &&
			    disk.minor == 0)
				continue;
			if ((disk.state & (1<<MD_DISK_SYNC))==0)
				continue;
			dv = map_dev(disk.major, disk.minor, 1);
			if (dv) {
				int fd2 = dev_open(dv, O_RDWR);
				if (fd2 < 0)
					continue;
				if (st->ss->load_super(st, fd2, NULL)==0) {
					if (st->ss->add_internal_bitmap(
						    st,
						    &chunk, delay, write_behind,
						    bitmapsize, 0, major)
						)
						st->ss->write_bitmap(st, fd2);
					else {
						fprintf(stderr, Name ": failed to create internal bitmap - chunksize problem.\n");
						close(fd2);
						return 1;
					}
				}
				close(fd2);
			}
		}
		array.state |= (1<<MD_SB_BITMAP_PRESENT);
		if (ioctl(fd, SET_ARRAY_INFO, &array)!= 0) {
			if (errno == EBUSY)
				fprintf(stderr, Name
					": Cannot add bitmap while array is"
					" resyncing or reshaping etc.\n");
			fprintf(stderr, Name ": failed to set internal bitmap.\n");
			return 1;
		}
	} else {
		int uuid[4];
		int bitmap_fd;
		int d;
		int max_devs = st->max_devs;

		/* try to load a superblock */
		for (d=0; d<max_devs; d++) {
			mdu_disk_info_t disk;
			char *dv;
			int fd2;
			disk.number = d;
			if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
				continue;
			if ((disk.major==0 && disk.minor==0) ||
			    (disk.state & (1<<MD_DISK_REMOVED)))
				continue;
			dv = map_dev(disk.major, disk.minor, 1);
			if (!dv) continue;
			fd2 = dev_open(dv, O_RDONLY);
			if (fd2 >= 0 &&
			    st->ss->load_super(st, fd2, NULL) == 0) {
				close(fd2);
				st->ss->uuid_from_super(st, uuid);
				break;
			}
			close(fd2);
		}
		if (d == max_devs) {
			fprintf(stderr, Name ": cannot find UUID for array!\n");
			return 1;
		}
		if (CreateBitmap(file, force, (char*)uuid, chunk,
				 delay, write_behind, bitmapsize, major)) {
			return 1;
		}
		bitmap_fd = open(file, O_RDWR);
		if (bitmap_fd < 0) {
			fprintf(stderr, Name ": weird: %s cannot be opened\n",
				file);
			return 1;
		}
		if (ioctl(fd, SET_BITMAP_FILE, bitmap_fd) < 0) {
			int err = errno;
			if (errno == EBUSY)
				fprintf(stderr, Name
					": Cannot add bitmap while array is"
					" resyncing or reshaping etc.\n");
			fprintf(stderr, Name ": Cannot set bitmap file for %s: %s\n",
				devname, strerror(err));
			return 1;
		}
	}

	return 0;
}


/*
 * When reshaping an array we might need to backup some data.
 * This is written to all spares with a 'super_block' describing it.
 * The superblock goes 4K from the end of the used space on the
 * device.
 * It if written after the backup is complete.
 * It has the following structure.
 */

static struct mdp_backup_super {
	char	magic[16];  /* md_backup_data-1 or -2 */
	__u8	set_uuid[16];
	__u64	mtime;
	/* start/sizes in 512byte sectors */
	__u64	devstart;	/* address on backup device/file of data */
	__u64	arraystart;
	__u64	length;
	__u32	sb_csum;	/* csum of preceeding bytes. */
	__u32   pad1;
	__u64	devstart2;	/* offset in to data of second section */
	__u64	arraystart2;
	__u64	length2;
	__u32	sb_csum2;	/* csum of preceeding bytes. */
	__u8 pad[512-68-32];
} __attribute__((aligned(512))) bsb, bsb2;

static __u32 bsb_csum(char *buf, int len)
{
	int i;
	int csum = 0;
	for (i=0; i<len; i++)
		csum = (csum<<3) + buf[0];
	return __cpu_to_le32(csum);
}

static int child_grow(int afd, struct mdinfo *sra, unsigned long blocks,
		      int *fds, unsigned long long *offsets,
		      int disks, int chunk, int level, int layout, int data,
		      int dests, int *destfd, unsigned long long *destoffsets);
static int child_shrink(int afd, struct mdinfo *sra, unsigned long blocks,
			int *fds, unsigned long long *offsets,
			int disks, int chunk, int level, int layout, int data,
			int dests, int *destfd, unsigned long long *destoffsets);
static int child_same_size(int afd, struct mdinfo *sra, unsigned long blocks,
			   int *fds, unsigned long long *offsets,
			   unsigned long long start,
			   int disks, int chunk, int level, int layout, int data,
			   int dests, int *destfd, unsigned long long *destoffsets);

static int check_idle(struct supertype *st)
{
	/* Check that all member arrays for this container, or the
	 * container of this array, are idle
	 */
	int container_dev = (st->container_dev != NoMdDev
			     ? st->container_dev : st->devnum);
	char container[40];
	struct mdstat_ent *ent, *e;
	int is_idle = 1;
	
	fmt_devname(container, container_dev);
	ent = mdstat_read(0, 0);
	for (e = ent ; e; e = e->next) {
		if (!is_container_member(e, container))
			continue;
		if (e->percent >= 0) {
			is_idle = 0;
			break;
		}
	}
	free_mdstat(ent);
	return is_idle;
}

static int freeze_container(struct supertype *st)
{
	int container_dev = (st->container_dev != NoMdDev
			     ? st->container_dev : st->devnum);
	char container[40];

	if (!check_idle(st))
		return -1;
	
	fmt_devname(container, container_dev);

	if (block_monitor(container, 1)) {
		fprintf(stderr, Name ": failed to freeze container\n");
		return -2;
	}

	return 1;
}

static void unfreeze_container(struct supertype *st)
{
	int container_dev = (st->container_dev != NoMdDev
			     ? st->container_dev : st->devnum);
	char container[40];
	
	fmt_devname(container, container_dev);

	unblock_monitor(container, 1);
}

static int freeze(struct supertype *st)
{
	/* Try to freeze resync/rebuild on this array/container.
	 * Return -1 if the array is busy,
	 * return -2 container cannot be frozen,
	 * return 0 if this kernel doesn't support 'frozen'
	 * return 1 if it worked.
	 */
	if (st->ss->external)
		return freeze_container(st);
	else {
		struct mdinfo *sra = sysfs_read(-1, st->devnum, GET_VERSION);
		int err;

		if (!sra)
			return -1;
		err = sysfs_freeze_array(sra);
		sysfs_free(sra);
		return err;
	}
}

static void unfreeze(struct supertype *st, int frozen)
{
	/* If 'frozen' is 1, unfreeze the array */
	if (frozen <= 0)
		return;

	if (st->ss->external)
		return unfreeze_container(st);
	else {
		struct mdinfo *sra = sysfs_read(-1, st->devnum, GET_VERSION);

		if (sra)
			sysfs_set_str(sra, NULL, "sync_action", "idle");
		else
			fprintf(stderr, Name ": failed to unfreeze array\n");
		sysfs_free(sra);
	}
}

static void wait_reshape(struct mdinfo *sra)
{
	int fd = sysfs_get_fd(sra, NULL, "sync_action");
	char action[20];

	if (fd < 0)
		return;

	while  (sysfs_fd_get_str(fd, action, 20) > 0 &&
		strncmp(action, "reshape", 7) == 0) {
		fd_set rfds;
		FD_ZERO(&rfds);
		FD_SET(fd, &rfds);
		select(fd+1, NULL, NULL, &rfds, NULL);
	}
	close(fd);
}

static int reshape_super(struct supertype *st, long long size, int level,
			 int layout, int chunksize, int raid_disks,
			 char *backup_file, char *dev, int verbose)
{
	/* nothing extra to check in the native case */
	if (!st->ss->external)
		return 0;
	if (!st->ss->reshape_super ||
	    !st->ss->manage_reshape) {
		fprintf(stderr, Name ": %s metadata does not support reshape\n",
			st->ss->name);
		return 1;
	}

	return st->ss->reshape_super(st, size, level, layout, chunksize,
				     raid_disks, backup_file, dev, verbose);
}

static void sync_metadata(struct supertype *st)
{
	if (st->ss->external) {
		if (st->update_tail)
			flush_metadata_updates(st);
		else
			st->ss->sync_metadata(st);
	}
}

static int subarray_set_num(char *container, struct mdinfo *sra, char *name, int n)
{
	/* when dealing with external metadata subarrays we need to be
	 * prepared to handle EAGAIN.  The kernel may need to wait for
	 * mdmon to mark the array active so the kernel can handle
	 * allocations/writeback when preparing the reshape action
	 * (md_allow_write()).  We temporarily disable safe_mode_delay
	 * to close a race with the array_state going clean before the
	 * next write to raid_disks / stripe_cache_size
	 */
	char safe[50];
	int rc;

	/* only 'raid_disks' and 'stripe_cache_size' trigger md_allow_write */
	if (strcmp(name, "raid_disks") != 0 &&
	    strcmp(name, "stripe_cache_size") != 0)
		return sysfs_set_num(sra, NULL, name, n);

	rc = sysfs_get_str(sra, NULL, "safe_mode_delay", safe, sizeof(safe));
	if (rc <= 0)
		return -1;
	sysfs_set_num(sra, NULL, "safe_mode_delay", 0);
	rc = sysfs_set_num(sra, NULL, name, n);
	if (rc < 0 && errno == EAGAIN) {
		ping_monitor(container);
		/* if we get EAGAIN here then the monitor is not active
		 * so stop trying
		 */
		rc = sysfs_set_num(sra, NULL, name, n);
	}
	sysfs_set_str(sra, NULL, "safe_mode_delay", safe);
	return rc;
}

static int reshape_container_raid_disks(char *container, int raid_disks)
{
	/* for each subarray switch to a raid level that can
	 * support the reshape, and set raid disks
	 */
	struct mdstat_ent *ent, *e;
	int changed = 0, rv = 0, err = 0;

	ent = mdstat_read(1, 0);
	if (!ent) {
		fprintf(stderr, Name ": unable to read /proc/mdstat\n");
		return -1;
	}

	changed = 0;
	for (e = ent; e; e = e->next) {
		struct mdinfo *sub;
		unsigned int cache;
		int level, takeover_delta = 0;

		if (!is_container_member(e, container))
			continue;

		level = map_name(pers, e->level);
		if (level == 0) {
			sub = sysfs_read(-1, e->devnum, GET_VERSION);
			if (!sub)
				break;
			/* metadata records 'orig_level' */
			rv = sysfs_set_num(sub, NULL, "level", 4);
			if (rv < 0) {
				err = errno;
				break;
			}
			/* we want spares to be used for capacity
			 * expansion, not rebuild
			 */
			takeover_delta = 1;

			sysfs_free(sub);
			level = 4;
		}

		sub = NULL;
		switch (level) {
		default:
			rv = -1;
			break;
		case 4:
		case 5:
		case 6:
			sub = sysfs_read(-1, e->devnum, GET_CHUNK|GET_CACHE);
			if (!sub)
				break;
			cache = (sub->array.chunk_size / 4096) * 4;
			if (cache > sub->cache_size)
				rv = subarray_set_num(container, sub,
						      "stripe_cache_size", cache);
			if (rv) {
				err = errno;
				break;
			}
			/* fall through */
		case 1:
			if (!sub)
				sub = sysfs_read(-1, e->devnum, GET_VERSION);
			if (!sub)
				break;

			rv = subarray_set_num(container, sub, "raid_disks",
					      raid_disks + takeover_delta);
			if (rv)
				err = errno;
			else
				changed++;
			break;
		}
		sysfs_free(sub);
		if (rv)
			break;
	}
	free_mdstat(ent);
	if (rv) {
		fprintf(stderr, Name
			": failed to initiate container reshape%s%s\n",
			err ? ": " : "", err ? strerror(err) : "");
		return rv;
	}

	return changed;
}

static void revert_container_raid_disks(struct supertype *st, int fd, char *container)
{
	/* we failed to prepare all subarrays in the container for
	 * reshape, so cancel the changes and restore the nominal raid
	 * level
	 */
	struct mdstat_ent *ent, *e;

	ent = mdstat_read(0, 0);
	if (!ent) {
		fprintf(stderr, Name
			": failed to read /proc/mdstat while aborting reshape\n");
		return;
	}

	if (st->ss->load_container(st, fd, NULL)) {
		fprintf(stderr, Name
			": failed read metadata while aborting reshape\n");
		return ;
	}


	for (e = ent; e; e = e->next) {
		int level_fixed = 0, disks_fixed = 0;
		struct mdinfo *sub, *prev;
		char *subarray;

		if (!is_container_member(e, container))
			continue;

		subarray = to_subarray(e, container);
		prev = st->ss->container_content(st, subarray);

		/* changing level might change raid_disks so we do it
		 * first and then check if raid_disks still needs fixing
		 */
		if (map_name(pers, e->level) != prev->array.level) {
			sub = sysfs_read(-1, e->devnum, GET_VERSION);
			if (sub &&
			    !sysfs_set_num(sub, NULL, "level", prev->array.level))
				level_fixed = 1;
			sysfs_free(sub);
		} else
			level_fixed = 1;

		sub = sysfs_read(-1, e->devnum, GET_DISKS);
		if (sub && sub->array.raid_disks != prev->array.raid_disks) {
			if (!subarray_set_num(container, sub, "raid_disks",
					      prev->array.raid_disks))
				disks_fixed = 1;
		} else if (sub)
			disks_fixed = 1;
		sysfs_free(sub);

		if (!disks_fixed || !level_fixed)
			fprintf(stderr, Name
				": failed to restore %s to a %d-disk %s array\n",
				e->dev, prev->array.raid_disks,
				map_num(pers, prev->array.level));
		free(prev);
	}
	st->ss->free_super(st);
	free_mdstat(ent);
}

int remove_disks_on_raid10_to_raid0_takeover(struct supertype *st,
					     struct mdinfo *sra,
					     int layout)
{
	int nr_of_copies;
	struct mdinfo *remaining;
	int slot;

	nr_of_copies = layout & 0xff;

	remaining = sra->devs;
	sra->devs = NULL;
	/* for each 'copy', select one device and remove from the list. */
	for (slot = 0; slot < sra->array.raid_disks; slot += nr_of_copies) {
		struct mdinfo **diskp;
		int found = 0;

		/* Find a working device to keep */
		for (diskp =  &remaining; *diskp ; diskp = &(*diskp)->next) {
			struct mdinfo *disk = *diskp;

			if (disk->disk.raid_disk < slot)
				continue;
			if (disk->disk.raid_disk >= slot + nr_of_copies)
				continue;
			if (disk->disk.state & (1<<MD_DISK_REMOVED))
				continue;
			if (disk->disk.state & (1<<MD_DISK_FAULTY))
				continue;
			if (!(disk->disk.state & (1<<MD_DISK_SYNC)))
				continue;

			/* We have found a good disk to use! */
			*diskp = disk->next;
			disk->next = sra->devs;
			sra->devs = disk;
			found = 1;
			break;
		}
		if (!found)
			break;
	}

	if (slot < sra->array.raid_disks) {
		/* didn't find all slots */
		struct mdinfo **e;
		e = &remaining;
		while (*e)
			e = &(*e)->next;
		*e = sra->devs;
		sra->devs = remaining;
		return 1;
	}

	/* Remove all 'remaining' devices from the array */
	while (remaining) {
		struct mdinfo *sd = remaining;
		remaining = sd->next;

		sysfs_set_str(sra, sd, "state", "faulty");
		sysfs_set_str(sra, sd, "slot", "none");
		sysfs_set_str(sra, sd, "state", "remove");
		sd->disk.state |= (1<<MD_DISK_REMOVED);
		sd->disk.state &= ~(1<<MD_DISK_SYNC);
		sd->next = sra->devs;
		sra->devs = sd;
	}
	return 0;
}

void reshape_free_fdlist(int *fdlist,
			 unsigned long long *offsets,
			 int size)
{
	int i;

	for (i = 0; i < size; i++)
		if (fdlist[i] >= 0)
			close(fdlist[i]);

	free(fdlist);
	free(offsets);
}

int reshape_prepare_fdlist(char *devname,
			   struct mdinfo *sra,
			   int raid_disks,
			   int nrdisks,
			   unsigned long blocks,
			   char *backup_file,
			   int *fdlist,
			   unsigned long long *offsets)
{
	int d = 0;
	struct mdinfo *sd;

	for (d = 0; d <= nrdisks; d++)
		fdlist[d] = -1;
	d = raid_disks;
	for (sd = sra->devs; sd; sd = sd->next) {
		if (sd->disk.state & (1<<MD_DISK_FAULTY))
			continue;
		if (sd->disk.state & (1<<MD_DISK_SYNC)) {
			char *dn = map_dev(sd->disk.major,
					   sd->disk.minor, 1);
			fdlist[sd->disk.raid_disk]
				= dev_open(dn, O_RDONLY);
			offsets[sd->disk.raid_disk] = sd->data_offset*512;
			if (fdlist[sd->disk.raid_disk] < 0) {
				fprintf(stderr,
					Name ": %s: cannot open component %s\n",
					devname, dn ? dn : "-unknown-");
				d = -1;
				goto release;
			}
		} else if (backup_file == NULL) {
			/* spare */
			char *dn = map_dev(sd->disk.major,
					   sd->disk.minor, 1);
				fdlist[d] = dev_open(dn, O_RDWR);
				offsets[d] = (sd->data_offset + sra->component_size - blocks - 8)*512;
				if (fdlist[d] < 0) {
					fprintf(stderr, Name ": %s: cannot open component %s\n",
						devname, dn ? dn : "-unknown-");
					d = -1;
					goto release;
				}
				d++;
			}
		}
release:
	return d;
}

int reshape_open_backup_file(char *backup_file,
			     int fd,
			     char *devname,
			     long blocks,
			     int *fdlist,
			     unsigned long long *offsets)
{
	/* Return 1 on success, 0 on any form of failure */
	/* need to check backup file is large enough */
	char buf[512];
	struct stat stb;
	unsigned int dev;
	int i;

	*fdlist = open(backup_file, O_RDWR|O_CREAT|O_EXCL,
		       S_IRUSR | S_IWUSR);
	*offsets = 8 * 512;
	if (*fdlist < 0) {
		fprintf(stderr, Name ": %s: cannot create backup file %s: %s\n",
			devname, backup_file, strerror(errno));
		return 0;
	}
	/* Guard against backup file being on array device.
	 * If array is partitioned or if LVM etc is in the
	 * way this will not notice, but it is better than
	 * nothing.
	 */
	fstat(*fdlist, &stb);
	dev = stb.st_dev;
	fstat(fd, &stb);
	if (stb.st_rdev == dev) {
		fprintf(stderr, Name ": backup file must NOT be"
			" on the array being reshaped.\n");
		close(*fdlist);
		return 0;
	}

	memset(buf, 0, 512);
	for (i=0; i < blocks + 1 ; i++) {
		if (write(*fdlist, buf, 512) != 512) {
			fprintf(stderr, Name ": %s: cannot create"
				" backup file %s: %s\n",
				devname, backup_file, strerror(errno));
			return 0;
		}
	}
	if (fsync(*fdlist) != 0) {
		fprintf(stderr, Name ": %s: cannot create backup file %s: %s\n",
			devname, backup_file, strerror(errno));
		return 0;
	}

	return 1;
}

unsigned long compute_backup_blocks(int nchunk, int ochunk,
				    unsigned int ndata, unsigned int odata)
{
	unsigned long a, b, blocks;
	/* So how much do we need to backup.
	 * We need an amount of data which is both a whole number of
	 * old stripes and a whole number of new stripes.
	 * So LCM for (chunksize*datadisks).
	 */
	a = (ochunk/512) * odata;
	b = (nchunk/512) * ndata;
	/* Find GCD */
	while (a != b) {
		if (a < b)
			b -= a;
		if (b < a)
			a -= b;
	}
	/* LCM == product / GCD */
	blocks = (ochunk/512) * (nchunk/512) * odata * ndata / a;

	return blocks;
}


int Grow_reshape(char *devname, int fd, int quiet, char *backup_file,
		 long long size,
		 int level, char *layout_str, int chunksize, int raid_disks,
		 int force)
{
	/* Make some changes in the shape of an array.
	 * The kernel must support the change.
	 *
	 * There are three different changes.  Each can trigger
	 * a resync or recovery so we freeze that until we have
	 * requested everything (if kernel supports freezing - 2.6.30).
	 * The steps are:
	 *  - change size (i.e. component_size)
	 *  - change level
	 *  - change layout/chunksize/ndisks
	 *
	 * The last can require a reshape.  It is different on different
	 * levels so we need to check the level before actioning it.
	 * Some times the level change needs to be requested after the
	 * reshape (e.g. raid6->raid5, raid5->raid0)
	 *
	 */
	struct mdu_array_info_s array, orig;
	char *c;
	int rv = 0;
	struct supertype *st;
	char *subarray = NULL;

	int nchunk, ochunk;
	int nlayout, olayout;
	int ndisks, odisks;
	unsigned int ndata, odata;
	int orig_level = UnSet;
	char alt_layout[40];
	int *fdlist;
	unsigned long long *offsets;
	int d;
	int nrdisks;
	int err;
	int frozen;
	unsigned long blocks, stripes;
	unsigned long cache;
	unsigned long long array_size;
	int changed = 0;
	char *container = NULL;
	int cfd = -1;
	int done;

	struct mdinfo *sra;
	struct mdinfo *sd;

	if (ioctl(fd, GET_ARRAY_INFO, &array) < 0) {
		fprintf(stderr, Name ": %s is not an active md array - aborting\n",
			devname);
		return 1;
	}

	if (size >= 0 &&
	    (chunksize || level!= UnSet || layout_str || raid_disks)) {
		fprintf(stderr, Name ": cannot change component size at the same time "
			"as other changes.\n"
			"   Change size first, then check data is intact before "
			"making other changes.\n");
		return 1;
	}

	if (raid_disks && raid_disks < array.raid_disks && array.level > 1 &&
	    get_linux_version() < 2006032 &&
	    !check_env("MDADM_FORCE_FEWER")) {
		fprintf(stderr, Name ": reducing the number of devices is not safe before Linux 2.6.32\n"
			"       Please use a newer kernel\n");
		return 1;
	}

	st = super_by_fd(fd, &subarray);
	if (!st) {
		fprintf(stderr, Name ": Unable to determine metadata format for %s\n", devname);
		return 1;
	}
	if (raid_disks > st->max_devs) {
		fprintf(stderr, Name ": Cannot increase raid-disks on this array"
			" beyond %d\n", st->max_devs);
		return 1;
	}

	/* in the external case we need to check that the requested reshape is
	 * supported, and perform an initial check that the container holds the
	 * pre-requisite spare devices (mdmon owns final validation)
	 */
	if (st->ss->external) {
		int container_dev;
		int rv;

		if (subarray) {
			container_dev = st->container_dev;
			cfd = open_dev_excl(st->container_dev);
		} else if (size >= 0 || layout_str != NULL || chunksize != 0 ||
			   level != UnSet) {
			fprintf(stderr,
				Name ": %s is a container, only 'raid-devices' can be changed\n",
				devname);
			return 1;
		} else {
			container_dev = st->devnum;
			close(fd);
			cfd = open_dev_excl(st->devnum);
			fd = cfd;
		}
		if (cfd < 0) {
			fprintf(stderr, Name ": Unable to open container for %s\n",
				devname);
			free(subarray);
			return 1;
		}

		container = devnum2devname(st->devnum);
		if (!container) {
			fprintf(stderr, Name ": Could not determine container name\n");
			free(subarray);
			return 1;
		}

		if (subarray)
			rv = st->ss->load_container(st, cfd, NULL);
		else
			rv = st->ss->load_super(st, cfd, NULL);
		if (rv) {
			fprintf(stderr, Name ": Cannot read superblock for %s\n",
				devname);
			free(subarray);
			return 1;
		}

		if (mdmon_running(container_dev))
			st->update_tail = &st->updates;
	} 

	if (raid_disks > array.raid_disks &&
	    array.spare_disks < (raid_disks - array.raid_disks) &&
	    !force) {
		fprintf(stderr,
			Name ": Need %d spare%s to avoid degraded array,"
			" and only have %d.\n"
			"       Use --force to over-ride this check.\n",
			raid_disks - array.raid_disks, 
			raid_disks - array.raid_disks == 1 ? "" : "s", 
			array.spare_disks);
		return 1;
	}

	sra = sysfs_read(fd, 0, GET_LEVEL | GET_DISKS | GET_DEVS | GET_STATE);
	if (sra) {
		if (st->ss->external && subarray == NULL) {
			array.level = LEVEL_CONTAINER;
			sra->array.level = LEVEL_CONTAINER;
		}
	} else {
		fprintf(stderr, Name ": failed to read sysfs parameters for %s\n",
			devname);
		return 1;
	}
	frozen = freeze(st);
	if (frozen < -1) {
		/* freeze() already spewed the reason */
		return 1;
	} else if (frozen < 0) {
		fprintf(stderr, Name ": %s is performing resync/recovery and cannot"
			" be reshaped\n", devname);
		return 1;
	}

	/* ========= set size =============== */
	if (size >= 0 && (size == 0 || size != array.size)) {
		long long orig_size = array.size;

		if (reshape_super(st, size, UnSet, UnSet, 0, 0, NULL, devname, !quiet)) {
			rv = 1;
			goto release;
		}
		sync_metadata(st);
		array.size = size;
		if (array.size != size) {
			/* got truncated to 32bit, write to
			 * component_size instead
			 */
			if (sra)
				rv = sysfs_set_num(sra, NULL,
						   "component_size", size);
			else
				rv = -1;
		} else
			rv = ioctl(fd, SET_ARRAY_INFO, &array);
		if (rv != 0) {
			int err = errno;

			/* restore metadata */
			if (reshape_super(st, orig_size, UnSet, UnSet, 0, 0,
					  NULL, devname, !quiet) == 0)
				sync_metadata(st);
			fprintf(stderr, Name ": Cannot set device size for %s: %s\n",
				devname, strerror(err));
			if (err == EBUSY && 
			    (array.state & (1<<MD_SB_BITMAP_PRESENT)))
				fprintf(stderr, "       Bitmap must be removed before size can be changed\n");
			rv = 1;
			goto release;
		}
		ioctl(fd, GET_ARRAY_INFO, &array);
		size = get_component_size(fd)/2;
		if (size == 0)
			size = array.size;
		if (!quiet)
			fprintf(stderr, Name ": component size of %s has been set to %lluK\n",
				devname, size);
		changed = 1;
	} else if (array.level != LEVEL_CONTAINER) {
		size = get_component_size(fd)/2;
		if (size == 0)
			size = array.size;
	}

	/* ========= check for Raid10 -> Raid0 conversion ===============
	 * current implemenation assumes that following conditions must be met:
	 * - far_copies == 1
	 * - near_copies == 2
	 */
	if (level == 0 && array.level == 10 &&
	    array.layout == ((1 << 8) + 2) && !(array.raid_disks & 1)) {
		int err;
		err = remove_disks_on_raid10_to_raid0_takeover(st, sra, array.layout);
		if (err) {
			dprintf(Name": Array cannot be reshaped\n");
			if (container)
				free(container);
			if (cfd > -1)
				close(cfd);
			return 1;
		}
	}

	/* ======= set level =========== */
	if (level != UnSet && level != array.level) {
		/* Trying to change the level.
		 * We might need to change layout first and schedule a
		 * level change for later.
		 * Level changes that can happen immediately are:
		 * 0->4,5,6  1->5  4->5,6  5->1,6
		 * Level changes that need a layout change first are:
		 * 6->5,4,0 : need a -6 layout, or parity-last
		 * 5->4,0   : need parity-last
		 */
		if ((array.level == 6 || array.level == 5) &&
		    (level == 5 || level == 4 || level == 0)) {
			/* Don't change level yet, but choose intermediate
			 * layout
			 */
			if (level == 5) {
				if (layout_str == NULL)
					switch (array.layout) {
					case ALGORITHM_LEFT_ASYMMETRIC:
					case ALGORITHM_LEFT_ASYMMETRIC_6:
					case ALGORITHM_ROTATING_N_RESTART:
						layout_str = "left-asymmetric-6";
						break;
					case ALGORITHM_LEFT_SYMMETRIC:
					case ALGORITHM_LEFT_SYMMETRIC_6:
					case ALGORITHM_ROTATING_N_CONTINUE:
						layout_str = "left-symmetric-6";
						break;
					case ALGORITHM_RIGHT_ASYMMETRIC:
					case ALGORITHM_RIGHT_ASYMMETRIC_6:
					case ALGORITHM_ROTATING_ZERO_RESTART:
						layout_str = "right-asymmetric-6";
						break;
					case ALGORITHM_RIGHT_SYMMETRIC:
					case ALGORITHM_RIGHT_SYMMETRIC_6:
						layout_str = "right-symmetric-6";
						break;
					case ALGORITHM_PARITY_0:
					case ALGORITHM_PARITY_0_6:
						layout_str = "parity-first-6";
						break;
					case ALGORITHM_PARITY_N:
						layout_str = "parity-last";
						break;
					default:
						fprintf(stderr, Name ": %s: cannot"
							"convert layout to RAID5 equivalent\n",
							devname);
						rv = 1;
						goto release;
					}
				else {
					int l = map_name(r5layout, layout_str);
					if (l == UnSet) {
						fprintf(stderr, Name ": %s: layout '%s' not recognised\n",
							devname, layout_str);
						rv = 1;
						goto release;
					}
					if (l != ALGORITHM_PARITY_N) {
						/* need the -6 version */
						char *ls = map_num(r5layout, l);
						strcat(strcpy(alt_layout, ls),
						       "-6");
						layout_str = alt_layout;
					}
				}
				if (raid_disks)
					/* The final raid6->raid5 conversion
					 * will reduce the number of disks,
					 * so now we need to aim higher
					 */
					raid_disks++;
			} else
				layout_str = "parity-last";
		} else {
			/* Level change is a simple takeover.  In the external
			 * case we don't check with the metadata handler until
			 * we establish what the final layout will be.  If the
			 * level change is disallowed we will revert to
			 * orig_level without disturbing the metadata, otherwise
			 * we will send an update.
			 */
			c = map_num(pers, level);
			if (c == NULL) {
				rv = 1;/* not possible */
				goto release;
			}
			if (!force) {
				/* Need to check there are enough spares */
				int spares_needed = 0;
				switch (array.level * 16 + level) {
				case 0x05:
					spares_needed = 1; break;
				case 0x06:
					spares_needed = 2; break;
				case 0x15:
					spares_needed = 1; break;
				case 0x16:
					spares_needed = 2; break;
				case 0x56:
					spares_needed = 1; break;
				}
				if (raid_disks > array.raid_disks)
					spares_needed += raid_disks-array.raid_disks;
				if (spares_needed > array.spare_disks) {
					fprintf(stderr,
						Name ": Need %d spare%s to avoid"
						" degraded array, and only have %d.\n"
						"       Use --force to over-ride"
						" this check.\n",
						spares_needed,
						spares_needed == 1 ? "" : "s", 
						array.spare_disks);
					rv = 1;
					goto release;
				}
			}
			err = sysfs_set_str(sra, NULL, "level", c);
			if (err) {
				err = errno;
				fprintf(stderr, Name ": %s: could not set level to %s\n",
					devname, c);
				if (err == EBUSY && 
				    (array.state & (1<<MD_SB_BITMAP_PRESENT)))
					fprintf(stderr, "       Bitmap must be removed before level can be changed\n");
				rv = 1;
				goto release;
			}
			orig = array;
			orig_level = orig.level;
			ioctl(fd, GET_ARRAY_INFO, &array);
			if (layout_str == NULL &&
			    orig.level == 5 && level == 6 &&
			    array.layout != orig.layout)
				layout_str = map_num(r5layout, orig.layout);
			if (!quiet)
				fprintf(stderr, Name " level of %s changed to %s\n",
					devname, c);
			changed = 1;
		}
	}

	/* ========= set shape (chunk_size / layout / ndisks)  ============== */
	/* Check if layout change is a no-op */
	switch (array.level) {
	case 5:
		if (layout_str && array.layout == map_name(r5layout, layout_str))
			layout_str = NULL;
		break;
	case 6:
		if (layout_str == NULL &&
		    ((chunksize && chunksize * 1024 != array.chunk_size) ||
		     (raid_disks && raid_disks != array.raid_disks)) &&
		    array.layout >= 16) {
			fprintf(stderr, Name
				": %s has a non-standard layout.  If you wish to preserve this\n"
				"      during the reshape, please specify --layout=preserve\n"
				"      If you want to change it, specify a layout or use --layout=normalise\n",
				devname);
			rv = 1;
			goto release;
		}
		if (layout_str &&
		    (strcmp(layout_str, "normalise") == 0 ||
		     strcmp(layout_str, "normalize") == 0)) {
			char *hyphen;
			strcpy(alt_layout, map_num(r6layout, array.layout));
			hyphen = strrchr(alt_layout, '-');
			if (hyphen && strcmp(hyphen, "-6") == 0) {
				*hyphen = 0;
				layout_str = alt_layout;
			}
		}

		if (layout_str && array.layout == map_name(r6layout, layout_str))
			layout_str = NULL;
		if (layout_str && strcmp(layout_str, "preserve") == 0)
			layout_str = NULL;
		break;
	}
	if (layout_str == NULL
	    && (chunksize == 0 || chunksize*1024 == array.chunk_size)
	    && (raid_disks == 0 || raid_disks == array.raid_disks)) {
		if (reshape_super(st, -1, level, UnSet, 0, 0, NULL, devname, !quiet)) {
			rv = 1;
			goto release;
		}
		sync_metadata(st);
		rv = 0;
		if (level != UnSet && level != array.level) {
			/* Looks like this level change doesn't need
			 * a reshape after all.
			 */
			c = map_num(pers, level);
			if (c) {
				rv = sysfs_set_str(sra, NULL, "level", c);
				if (rv) {
					int err = errno;
					fprintf(stderr, Name ": %s: could not set level to %s\n",
						devname, c);
					if (err == EBUSY && 
					    (array.state & (1<<MD_SB_BITMAP_PRESENT)))
						fprintf(stderr, "       Bitmap must be removed before level can be changed\n");
					rv = 1;
				}
			}
		} else if (!changed && !quiet)
			fprintf(stderr, Name ": %s: no change requested\n",
				devname);

		if (st->ss->external && !mdmon_running(st->container_dev) &&
		    level > 0) {
			start_mdmon(st->container_dev);
			ping_monitor(container);
		}
		goto release;
	}

	c = map_num(pers, array.level);
	if (c == NULL) c = "-unknown-";
	switch (array.level) {
	default: /* raid0, linear, multipath cannot be reconfigured */
		fprintf(stderr, Name ": %s array %s cannot be reshaped.\n",
			c, devname);
		/* TODO raid0 raiddisks can be reshaped via raid4 */
		rv = 1;
		break;
	case LEVEL_CONTAINER: {
		int count;

		/* double check that we are not changing anything but raid_disks */
		if (size >= 0 || layout_str != NULL || chunksize != 0 || level != UnSet) {
			fprintf(stderr,
				Name ": %s is a container, only 'raid-devices' can be changed\n",
				devname);
			rv = 1;
			goto release;
		}

		st->update_tail = &st->updates;
		if (reshape_super(st, -1, UnSet, UnSet, 0, raid_disks,
				  backup_file, devname, !quiet)) {
			rv = 1;
			goto release;
		}

		count = reshape_container_raid_disks(container, raid_disks);
		if (count < 0) {
			revert_container_raid_disks(st, fd, container);
			rv = 1;
			goto release;
		} else if (count == 0) {
			if (!quiet)
				fprintf(stderr, Name
					": no active subarrays to reshape\n");
			goto release;
		}

		if (!mdmon_running(st->devnum)) {
			start_mdmon(st->devnum);
			ping_monitor(container);
		}
		sync_metadata(st);

		/* give mdmon a chance to allocate spares */
		ping_manager(container);

		/* manage_reshape takes care of releasing the array(s) */
		st->ss->manage_reshape(st, backup_file);
		frozen = 0;
		goto release;
	}
	case LEVEL_FAULTY: /* only 'layout' change is permitted */

		if (chunksize  || raid_disks) {
			fprintf(stderr, Name ": %s: Cannot change chunksize or disks of a 'faulty' array\n",
				devname);
			rv = 1;
			break;
		}
		if (layout_str == NULL)
			break; /* nothing to do.... */

		array.layout = parse_layout_faulty(layout_str);
		if (array.layout < 0) {
			fprintf(stderr, Name ": %s: layout %s not understood for 'faulty' array\n",
				devname, layout_str);
			rv = 1;
			break;
		}
		if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
			fprintf(stderr, Name ": Cannot set layout for %s: %s\n",
				devname, strerror(errno));
			rv = 1;
		} else if (!quiet)
			printf("layout for %s set to %d\n", devname, array.layout);
		break;

	case 1: /* only raid_disks can each be changed. */

		if (chunksize || layout_str != NULL) {
			fprintf(stderr, Name ": %s: Cannot change chunk size or layout for a RAID1 array.\n",
				devname);
			rv = 1;
			break;
		}
		if (raid_disks > 0) {
			if (reshape_super(st, -1, UnSet, UnSet, 0, raid_disks,
					  NULL, devname, !quiet)) {
				rv = 1;
				goto release;
			}
			sync_metadata(st);
			array.raid_disks = raid_disks;
			if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
				fprintf(stderr, Name ": Cannot set raid-devices for %s: %s\n",
					devname, strerror(errno));
				rv = 1;
			}
		}
		break;

	case 4:
	case 5:
	case 6:

		/*
		 * layout/chunksize/raid_disks can be changed
		 * though the kernel may not support it all.
		 */
		if (subarray) {
			fprintf(stderr, Name ": Cannot reshape subarrays yet\n");
			break;
		}

		/*
		 * There are three possibilities.
		 * 1/ The array will shrink.
		 *    We need to ensure the reshape will pause before reaching
		 *    the 'critical section'.  We also need to fork and wait for
		 *    that to happen.  When it does we 
		 *       suspend/backup/complete/unfreeze
		 *
		 * 2/ The array will not change size.
		 *    This requires that we keep a backup of a sliding window
		 *    so that we can restore data after a crash.  So we need
		 *    to fork and monitor progress.
		 *
		 * 3/ The array will grow. This is relatively easy.
		 *    However the kernel's restripe routines will cheerfully
		 *    overwrite some early data before it is safe.  So we
		 *    need to make a backup of the early parts of the array
		 *    and be ready to restore it if rebuild aborts very early.
		 *
		 *    We backup data by writing it to one spare, or to a
		 *    file which was given on command line.
		 *
		 *    [FOLLOWING IS OLD AND PARTLY WRONG]
		 *    So: we enumerate the devices in the array and
		 *    make sure we can open all of them.
		 *    Then we freeze the early part of the array and
		 *    backup to the various spares.
		 *    Then we request changes and start the reshape.
		 *    Monitor progress until it has passed the danger zone.
		 *    and finally invalidate the copied data and unfreeze the
		 *    start of the array.
		 *
		 * In each case, we first make sure that storage is available
		 * for the required backup.
		 * Then we:
		 *   -  request the shape change.
		 *   -  for to handle backup etc.
		 */
		nchunk = ochunk = array.chunk_size;
		nlayout = olayout = array.layout;
		ndisks = odisks = array.raid_disks;

		if (chunksize) {
			nchunk = chunksize * 1024;
			if (size % chunksize) {
				fprintf(stderr, Name ": component size %lluK is not"
					" a multiple of chunksize %dK\n",
					size, chunksize);
				break;
			}
		}
		if (layout_str != NULL)
			switch(array.level) {
			case 4: /* ignore layout */
				break;
			case 5:
				nlayout = map_name(r5layout, layout_str);
				if (nlayout == UnSet) {
					fprintf(stderr, Name ": layout %s not understood for raid5.\n",
						layout_str);
					rv = 1;
					goto release;
				}
				break;

			case 6:
				nlayout = map_name(r6layout, layout_str);
				if (nlayout == UnSet) {
					fprintf(stderr, Name ": layout %s not understood for raid6.\n",
						layout_str);
					rv = 1;
					goto release;
				}
				break;
			}
		if (raid_disks) ndisks = raid_disks;

		odata = odisks-1;
		ndata = ndisks-1;
		if (array.level == 6) {
			odata--; /* number of data disks */
			ndata--;
		}

		if (odata == ndata &&
		    get_linux_version() < 2006032) {
			fprintf(stderr, Name ": in-place reshape is not safe before 2.6.32, sorry.\n");
			break;
		}

		/* Check that we can hold all the data */
		get_dev_size(fd, NULL, &array_size);
		if (ndata * (unsigned long long)size < (array_size/1024)) {
			fprintf(stderr, Name ": this change will reduce the size of the array.\n"
				"       use --grow --array-size first to truncate array.\n"
				"       e.g. mdadm --grow %s --array-size %llu\n",
				devname, ndata * size);
			rv = 1;
			break;
		}

		blocks = compute_backup_blocks(nchunk, ochunk, ndata, odata);

		sysfs_free(sra);
		sra = sysfs_read(fd, 0,
				 GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE|
				 GET_CACHE);

		if (!sra) {
			fprintf(stderr, Name ": %s: Cannot get array details from sysfs\n",
				devname);
			rv = 1;
			break;
		}

		if (ndata == odata) {
			/* Make 'blocks' bigger for better throughput, but
			 * not so big that we reject it below.
			 * Try for 16 megabytes
			 */
			while (blocks * 32 < sra->component_size &&
			       blocks < 16*1024*2)
			       blocks *= 2;
		} else
			fprintf(stderr, Name ": Need to backup %luK of critical "
				"section..\n", blocks/2);

		if (blocks >= sra->component_size/2) {
			fprintf(stderr, Name ": %s: Something wrong - reshape aborted\n",
				devname);
			rv = 1;
			break;
		}
		nrdisks = array.raid_disks + sra->array.spare_disks;
		/* Now we need to open all these devices so we can read/write.
		 */
		fdlist = malloc((1+nrdisks) * sizeof(int));
		offsets = malloc((1+nrdisks) * sizeof(offsets[0]));
		if (!fdlist || !offsets) {
			fprintf(stderr, Name ": malloc failed: grow aborted\n");
			rv = 1;
			break;
		}

		d = reshape_prepare_fdlist(devname, sra, array.raid_disks,
					   nrdisks, blocks, backup_file,
					   fdlist, offsets);
		if (d < 0) {
			rv = 1;
			goto release;
		}
		if (backup_file == NULL) {
			if (st->ss->external && !st->ss->manage_reshape) {
				fprintf(stderr, Name ": %s Grow operation not supported by %s metadata\n",
					devname, st->ss->name);
				rv = 1;
				break;
			}
			if (ndata <= odata) {
				fprintf(stderr, Name ": %s: Cannot grow - need backup-file\n",
					devname);
				rv = 1;
				break;
			} else if (sra->array.spare_disks == 0) {
				fprintf(stderr, Name ": %s: Cannot grow - need a spare or "
					"backup-file to backup critical section\n",
					devname);
				rv = 1;
				break;
			}
			if (d == array.raid_disks) {
				fprintf(stderr, Name ": %s: No spare device for backup\n",
					devname);
				rv = 1;
				break;
			}
		} else {
			if (!reshape_open_backup_file(backup_file, fd, devname,
						      (signed)blocks,
						      fdlist+d, offsets+d)) {
				rv = 1;
				break;
			}
			d++;
		}

		/* check that the operation is supported by the metadata */
		if (reshape_super(st, -1, level, nlayout, nchunk, ndisks,
				  backup_file, devname, !quiet)) {
			rv = 1;
			break;
		}

		/* lastly, check that the internal stripe cache is
		 * large enough, or it won't work.
		 */
		
		cache = (nchunk < ochunk) ? ochunk : nchunk;
		cache = cache * 4 / 4096;
		if (cache < blocks / 8 / odisks + 16)
			/* Make it big enough to hold 'blocks' */
			cache = blocks / 8 / odisks + 16;
		if (sra->cache_size < cache)
			sysfs_set_num(sra, NULL, "stripe_cache_size",
				      cache+1);
		/* Right, everything seems fine. Let's kick things off.
		 * If only changing raid_disks, use ioctl, else use
		 * sysfs.
		 */
		sync_metadata(st);
		if (ochunk == nchunk && olayout == nlayout) {
			array.raid_disks = ndisks;
			if (ioctl(fd, SET_ARRAY_INFO, &array) != 0) {
				int err = errno;
				rv = 1;
				fprintf(stderr, Name ": Cannot set device shape for %s: %s\n",
					devname, strerror(errno));
				if (ndisks < odisks &&
				    get_linux_version() < 2006030)
					fprintf(stderr, Name ": linux 2.6.30 or later required\n");
				if (err == EBUSY && 
				    (array.state & (1<<MD_SB_BITMAP_PRESENT)))
					fprintf(stderr, "       Bitmap must be removed before shape can be changed\n");

				break;
			}
		} else {
			/* set them all just in case some old 'new_*' value
			 * persists from some earlier problem
			 */
			int err = err; /* only used if rv==1, and always set if
					* rv==1, so initialisation not needed,
					* despite gcc warning
					*/
			if (sysfs_set_num(sra, NULL, "chunk_size", nchunk) < 0)
				rv = 1, err = errno;
			if (!rv && sysfs_set_num(sra, NULL, "layout", nlayout) < 0)
				rv = 1, err = errno;
			if (!rv && sysfs_set_num(sra, NULL, "raid_disks", ndisks) < 0)
				rv = 1, err = errno;
			if (rv) {
				fprintf(stderr, Name ": Cannot set device shape for %s\n",
					devname);
				if (get_linux_version() < 2006030)
					fprintf(stderr, Name ": linux 2.6.30 or later required\n");
				if (err == EBUSY && 
				    (array.state & (1<<MD_SB_BITMAP_PRESENT)))
					fprintf(stderr, "       Bitmap must be removed before shape can be changed\n");
				break;
			}
		}

		if (ndisks == 2 && odisks == 2) {
			/* No reshape is needed in this trivial case */
			rv = 0;
			break;
		}

		if (st->ss->external) {
			/* metadata handler takes it from here */
			ping_manager(container);
			st->ss->manage_reshape(st, backup_file);
			frozen = 0;
			break;
		}

		/* set up the backup-super-block.  This requires the
		 * uuid from the array.
		 */
		/* Find a superblock */
		for (sd = sra->devs; sd; sd = sd->next) {
			char *dn;
			int devfd;
			int ok;
			if (sd->disk.state & (1<<MD_DISK_FAULTY))
				continue;
			dn = map_dev(sd->disk.major, sd->disk.minor, 1);
			devfd = dev_open(dn, O_RDONLY);
			if (devfd < 0)
				continue;
			ok = st->ss->load_super(st, devfd, NULL);
			close(devfd);
			if (ok >= 0)
				break;
		}
		if (!sd) {
			fprintf(stderr, Name ": %s: Cannot find a superblock\n",
				devname);
			rv = 1;
			break;
		}

		memset(&bsb, 0, 512);
		memcpy(bsb.magic, "md_backup_data-1", 16);
		st->ss->uuid_from_super(st, (int*)&bsb.set_uuid);
		bsb.mtime = __cpu_to_le64(time(0));
		bsb.devstart2 = blocks;
		stripes = blocks / (ochunk/512) / odata;
		/* Now we just need to kick off the reshape and watch, while
		 * handling backups of the data...
		 * This is all done by a forked background process.
		 */
		switch(fork()) {
		case 0:
			close(fd);
			if (check_env("MDADM_GROW_VERIFY"))
				fd = open(devname, O_RDONLY | O_DIRECT);
			else
				fd = -1;
			mlockall(MCL_FUTURE);

			if (odata < ndata)
				done = child_grow(fd, sra, stripes,
						  fdlist, offsets,
						  odisks, ochunk, array.level, olayout, odata,
						  d - odisks, fdlist+odisks, offsets+odisks);
			else if (odata > ndata)
				done = child_shrink(fd, sra, stripes,
						    fdlist, offsets,
						    odisks, ochunk, array.level, olayout, odata,
						    d - odisks, fdlist+odisks, offsets+odisks);
			else
				done = child_same_size(fd, sra, stripes,
						       fdlist, offsets,
						       0,
						       odisks, ochunk, array.level, olayout, odata,
						       d - odisks, fdlist+odisks, offsets+odisks);
			if (backup_file && done)
				unlink(backup_file);
			if (level != UnSet && level != array.level) {
				/* We need to wait for the reshape to finish
				 * (which will have happened unless odata < ndata)
				 * and then set the level
				 */

				c = map_num(pers, level);
				if (c == NULL)
					exit(0);/* not possible */

				if (odata < ndata)
					wait_reshape(sra);
				err = sysfs_set_str(sra, NULL, "level", c);
				if (err)
					fprintf(stderr, Name ": %s: could not set level to %s\n",
						devname, c);
			}
			exit(0);
		case -1:
			fprintf(stderr, Name ": Cannot run child to monitor reshape: %s\n",
				strerror(errno));
			rv = 1;
			break;
		default:
			/* The child will take care of unfreezing the array */
			frozen = 0;
			break;
		}
		break;

	}

 release:
	if (rv && orig_level != UnSet && sra) {
		c = map_num(pers, orig_level);
		if (c && sysfs_set_str(sra, NULL, "level", c) == 0)
			fprintf(stderr, Name ": aborting level change\n");
	}
	unfreeze(st, frozen);
	return rv;
}

/*
 * We run a child process in the background which performs the following
 * steps:
 *   - wait for resync to reach a certain point
 *   - suspend io to the following section
 *   - backup that section
 *   - allow resync to proceed further
 *   - resume io
 *   - discard the backup.
 *
 * When are combined in slightly different ways in the three cases.
 * Grow:
 *   - suspend/backup/allow/wait/resume/discard
 * Shrink:
 *   - allow/wait/suspend/backup/allow/wait/resume/discard
 * same-size:
 *   - wait/resume/discard/suspend/backup/allow
 *
 * suspend/backup/allow always come together
 * wait/resume/discard do too.
 * For the same-size case we have two backups to improve flow.
 * 
 */

/* FIXME return status is never checked */
static int grow_backup(struct mdinfo *sra,
		unsigned long long offset, /* per device */
		unsigned long stripes, /* per device */
		int *sources, unsigned long long *offsets,
		int disks, int chunk, int level, int layout,
		int dests, int *destfd, unsigned long long *destoffsets,
		int part, int *degraded,
		char *buf)
{
	/* Backup 'blocks' sectors at 'offset' on each device of the array,
	 * to storage 'destfd' (offset 'destoffsets'), after first
	 * suspending IO.  Then allow resync to continue
	 * over the suspended section.
	 * Use part 'part' of the backup-super-block.
	 */
	int odata = disks;
	int rv = 0;
	int i;
	unsigned long long ll;
	int new_degraded;
	//printf("offset %llu\n", offset);
	if (level >= 4)
		odata--;
	if (level == 6)
		odata--;
	sysfs_set_num(sra, NULL, "suspend_hi", (offset + stripes * (chunk/512)) * odata);
	/* Check that array hasn't become degraded, else we might backup the wrong data */
	sysfs_get_ll(sra, NULL, "degraded", &ll);
	new_degraded = (int)ll;
	if (new_degraded != *degraded) {
		/* check each device to ensure it is still working */
		struct mdinfo *sd;
		for (sd = sra->devs ; sd ; sd = sd->next) {
			if (sd->disk.state & (1<<MD_DISK_FAULTY))
				continue;
			if (sd->disk.state & (1<<MD_DISK_SYNC)) {
				char sbuf[20];
				if (sysfs_get_str(sra, sd, "state", sbuf, 20) < 0 ||
				    strstr(sbuf, "faulty") ||
				    strstr(sbuf, "in_sync") == NULL) {
					/* this device is dead */
					sd->disk.state = (1<<MD_DISK_FAULTY);
					if (sd->disk.raid_disk >= 0 &&
					    sources[sd->disk.raid_disk] >= 0) {
						close(sources[sd->disk.raid_disk]);
						sources[sd->disk.raid_disk] = -1;
					}
				}
			}
		}
		*degraded = new_degraded;
	}
	if (part) {
		bsb.arraystart2 = __cpu_to_le64(offset * odata);
		bsb.length2 = __cpu_to_le64(stripes * (chunk/512) * odata);
	} else {
		bsb.arraystart = __cpu_to_le64(offset * odata);
		bsb.length = __cpu_to_le64(stripes * (chunk/512) * odata);
	}
	if (part)
		bsb.magic[15] = '2';
	for (i = 0; i < dests; i++)
		if (part)
			lseek64(destfd[i], destoffsets[i] + __le64_to_cpu(bsb.devstart2)*512, 0);
		else
			lseek64(destfd[i], destoffsets[i], 0);

	rv = save_stripes(sources, offsets, 
			  disks, chunk, level, layout,
			  dests, destfd,
			  offset*512*odata, stripes * chunk * odata,
			  buf);

	if (rv)
		return rv;
	bsb.mtime = __cpu_to_le64(time(0));
	for (i = 0; i < dests; i++) {
		bsb.devstart = __cpu_to_le64(destoffsets[i]/512);

		bsb.sb_csum = bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb));
		if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0)
			bsb.sb_csum2 = bsb_csum((char*)&bsb,
						((char*)&bsb.sb_csum2)-((char*)&bsb));

		rv = -1;
		if ((unsigned long long)lseek64(destfd[i], destoffsets[i] - 4096, 0)
		    != destoffsets[i] - 4096)
			break;
		if (write(destfd[i], &bsb, 512) != 512)
			break;
		if (destoffsets[i] > 4096) {
			if ((unsigned long long)lseek64(destfd[i], destoffsets[i]+stripes*chunk*odata, 0) !=
			    destoffsets[i]+stripes*chunk*odata)
				break;
			if (write(destfd[i], &bsb, 512) != 512)
				break;
		}
		fsync(destfd[i]);
		rv = 0;
	}

	return rv;
}

/* in 2.6.30, the value reported by sync_completed can be
 * less that it should be by one stripe.
 * This only happens when reshape hits sync_max and pauses.
 * So allow wait_backup to either extent sync_max further
 * than strictly necessary, or return before the
 * sync has got quite as far as we would really like.
 * This is what 'blocks2' is for.
 * The various caller give appropriate values so that
 * every works.
 */
/* FIXME return value is often ignored */
static int wait_backup(struct mdinfo *sra,
		unsigned long long offset, /* per device */
		unsigned long long blocks, /* per device */
		unsigned long long blocks2, /* per device - hack */
		int dests, int *destfd, unsigned long long *destoffsets,
		int part)
{
	/* Wait for resync to pass the section that was backed up
	 * then erase the backup and allow IO
	 */
	int fd = sysfs_get_fd(sra, NULL, "sync_completed");
	unsigned long long completed;
	int i;
	int rv;

	if (fd < 0)
		return -1;
	sysfs_set_num(sra, NULL, "sync_max", offset + blocks + blocks2);
	if (offset == 0)
		sysfs_set_str(sra, NULL, "sync_action", "reshape");

	if (sysfs_fd_get_ll(fd, &completed) < 0) {
		close(fd);
		return -1;
	}
	while (completed < offset + blocks) {
		char action[20];
		fd_set rfds;
		FD_ZERO(&rfds);
		FD_SET(fd, &rfds);
		select(fd+1, NULL, NULL, &rfds, NULL);
		if (sysfs_fd_get_ll(fd, &completed) < 0) {
			close(fd);
			return -1;
		}
		if (sysfs_get_str(sra, NULL, "sync_action",
				  action, 20) > 0 &&
		    strncmp(action, "reshape", 7) != 0)
			break;
	}
	close(fd);

	if (part) {
		bsb.arraystart2 = __cpu_to_le64(0);
		bsb.length2 = __cpu_to_le64(0);
	} else {
		bsb.arraystart = __cpu_to_le64(0);
		bsb.length = __cpu_to_le64(0);
	}
	bsb.mtime = __cpu_to_le64(time(0));
	rv = 0;
	for (i = 0; i < dests; i++) {
		bsb.devstart = __cpu_to_le64(destoffsets[i]/512);
		bsb.sb_csum = bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb));
		if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0)
			bsb.sb_csum2 = bsb_csum((char*)&bsb,
						((char*)&bsb.sb_csum2)-((char*)&bsb));
		if ((unsigned long long)lseek64(destfd[i], destoffsets[i]-4096, 0) !=
		    destoffsets[i]-4096)
			rv = -1;
		if (rv == 0 && 
		    write(destfd[i], &bsb, 512) != 512)
			rv = -1;
		fsync(destfd[i]);
	}
	return rv;
}

static void fail(char *msg)
{
	int rv;
	rv = (write(2, msg, strlen(msg)) != (int)strlen(msg));
	rv |= (write(2, "\n", 1) != 1);
	exit(rv ? 1 : 2);
}

static char *abuf, *bbuf;
static unsigned long long abuflen;
static void validate(int afd, int bfd, unsigned long long offset)
{
	/* check that the data in the backup against the array.
	 * This is only used for regression testing and should not
	 * be used while the array is active
	 */
	if (afd < 0)
		return;
	lseek64(bfd, offset - 4096, 0);
	if (read(bfd, &bsb2, 512) != 512)
		fail("cannot read bsb");
	if (bsb2.sb_csum != bsb_csum((char*)&bsb2,
				     ((char*)&bsb2.sb_csum)-((char*)&bsb2)))
		fail("first csum bad");
	if (memcmp(bsb2.magic, "md_backup_data", 14) != 0)
		fail("magic is bad");
	if (memcmp(bsb2.magic, "md_backup_data-2", 16) == 0 &&
	    bsb2.sb_csum2 != bsb_csum((char*)&bsb2,
				     ((char*)&bsb2.sb_csum2)-((char*)&bsb2)))
		fail("second csum bad");

	if (__le64_to_cpu(bsb2.devstart)*512 != offset)
		fail("devstart is wrong");

	if (bsb2.length) {
		unsigned long long len = __le64_to_cpu(bsb2.length)*512;

		if (abuflen < len) {
			free(abuf);
			free(bbuf);
			abuflen = len;
			if (posix_memalign((void**)&abuf, 4096, abuflen) ||
			    posix_memalign((void**)&bbuf, 4096, abuflen)) {
				abuflen = 0;
				/* just stop validating on mem-alloc failure */
				return;
			}
		}

		lseek64(bfd, offset, 0);
		if ((unsigned long long)read(bfd, bbuf, len) != len) {
			//printf("len %llu\n", len);
			fail("read first backup failed");
		}
		lseek64(afd, __le64_to_cpu(bsb2.arraystart)*512, 0);
		if ((unsigned long long)read(afd, abuf, len) != len)
			fail("read first from array failed");
		if (memcmp(bbuf, abuf, len) != 0) {
			#if 0
			int i;
			printf("offset=%llu len=%llu\n",
			       (unsigned long long)__le64_to_cpu(bsb2.arraystart)*512, len);
			for (i=0; i<len; i++)
				if (bbuf[i] != abuf[i]) {
					printf("first diff byte %d\n", i);
					break;
				}
			#endif
			fail("data1 compare failed");
		}
	}
	if (bsb2.length2) {
		unsigned long long len = __le64_to_cpu(bsb2.length2)*512;

		if (abuflen < len) {
			free(abuf);
			free(bbuf);
			abuflen = len;
			abuf = malloc(abuflen);
			bbuf = malloc(abuflen);
		}

		lseek64(bfd, offset+__le64_to_cpu(bsb2.devstart2)*512, 0);
		if ((unsigned long long)read(bfd, bbuf, len) != len)
			fail("read second backup failed");
		lseek64(afd, __le64_to_cpu(bsb2.arraystart2)*512, 0);
		if ((unsigned long long)read(afd, abuf, len) != len)
			fail("read second from array failed");
		if (memcmp(bbuf, abuf, len) != 0)
			fail("data2 compare failed");
	}
}

static int child_grow(int afd, struct mdinfo *sra, unsigned long stripes,
		      int *fds, unsigned long long *offsets,
		      int disks, int chunk, int level, int layout, int data,
		      int dests, int *destfd, unsigned long long *destoffsets)
{
	char *buf;
	int degraded = 0;

	if (posix_memalign((void**)&buf, 4096, disks * chunk))
		/* Don't start the 'reshape' */
		return 0;
	sysfs_set_num(sra, NULL, "suspend_hi", 0);
	sysfs_set_num(sra, NULL, "suspend_lo", 0);
	grow_backup(sra, 0, stripes,
		    fds, offsets, disks, chunk, level, layout,
		    dests, destfd, destoffsets,
		    0, &degraded, buf);
	validate(afd, destfd[0], destoffsets[0]);
	wait_backup(sra, 0, stripes * (chunk / 512), stripes * (chunk / 512),
		    dests, destfd, destoffsets,
		    0);
	sysfs_set_num(sra, NULL, "suspend_lo", (stripes * (chunk/512)) * data);
	free(buf);
	/* FIXME this should probably be numeric */
	sysfs_set_str(sra, NULL, "sync_max", "max");
	return 1;
}

static int child_shrink(int afd, struct mdinfo *sra, unsigned long stripes,
			int *fds, unsigned long long *offsets,
			int disks, int chunk, int level, int layout, int data,
			int dests, int *destfd, unsigned long long *destoffsets)
{
	char *buf;
	unsigned long long start;
	int rv;
	int degraded = 0;

	if (posix_memalign((void**)&buf, 4096, disks * chunk))
		return 0;
	start = sra->component_size - stripes * (chunk/512);
	sysfs_set_num(sra, NULL, "sync_max", start);
	sysfs_set_str(sra, NULL, "sync_action", "reshape");
	sysfs_set_num(sra, NULL, "suspend_lo", 0);
	sysfs_set_num(sra, NULL, "suspend_hi", 0);
	rv = wait_backup(sra, 0, start - stripes * (chunk/512), stripes * (chunk/512),
			 dests, destfd, destoffsets, 0);
	if (rv < 0)
		return 0;
	grow_backup(sra, 0, stripes,
		    fds, offsets,
		    disks, chunk, level, layout,
		    dests, destfd, destoffsets,
		    0, &degraded, buf);
	validate(afd, destfd[0], destoffsets[0]);
	wait_backup(sra, start, stripes*(chunk/512), 0,
		    dests, destfd, destoffsets, 0);
	sysfs_set_num(sra, NULL, "suspend_lo", (stripes * (chunk/512)) * data);
	free(buf);
	/* FIXME this should probably be numeric */
	sysfs_set_str(sra, NULL, "sync_max", "max");
	return 1;
}

static int child_same_size(int afd, struct mdinfo *sra, unsigned long stripes,
			   int *fds, unsigned long long *offsets,
			   unsigned long long start,
			   int disks, int chunk, int level, int layout, int data,
			   int dests, int *destfd, unsigned long long *destoffsets)
{
	unsigned long long size;
	unsigned long tailstripes = stripes;
	int part;
	char *buf;
	unsigned long long speed;
	int degraded = 0;


	if (posix_memalign((void**)&buf, 4096, disks * chunk))
		return 0;

	sysfs_set_num(sra, NULL, "suspend_lo", 0);
	sysfs_set_num(sra, NULL, "suspend_hi", 0);

	sysfs_get_ll(sra, NULL, "sync_speed_min", &speed);
	sysfs_set_num(sra, NULL, "sync_speed_min", 200000);

	grow_backup(sra, start, stripes,
		    fds, offsets,
		    disks, chunk, level, layout,
		    dests, destfd, destoffsets,
		    0, &degraded, buf);
	grow_backup(sra, (start + stripes) * (chunk/512), stripes,
		    fds, offsets,
		    disks, chunk, level, layout,
		    dests, destfd, destoffsets,
		    1, &degraded, buf);
	validate(afd, destfd[0], destoffsets[0]);
	part = 0;
	start += stripes * 2; /* where to read next */
	size = sra->component_size / (chunk/512);
	while (start < size) {
		if (wait_backup(sra, (start-stripes*2)*(chunk/512),
				stripes*(chunk/512), 0,
				dests, destfd, destoffsets,
				part) < 0)
			return 0;
		sysfs_set_num(sra, NULL, "suspend_lo", start*(chunk/512) * data);
		if (start + stripes > size)
			tailstripes = (size - start);

		grow_backup(sra, start*(chunk/512), tailstripes,
			    fds, offsets,
			    disks, chunk, level, layout,
			    dests, destfd, destoffsets,
			    part, &degraded, buf);
		start += stripes;
		part = 1 - part;
		validate(afd, destfd[0], destoffsets[0]);
	}
	if (wait_backup(sra, (start-stripes*2) * (chunk/512), stripes * (chunk/512), 0,
			dests, destfd, destoffsets,
			part) < 0)
		return 0;
	sysfs_set_num(sra, NULL, "suspend_lo", ((start-stripes)*(chunk/512)) * data);
	wait_backup(sra, (start-stripes) * (chunk/512), tailstripes * (chunk/512), 0,
		    dests, destfd, destoffsets,
		    1-part);
	sysfs_set_num(sra, NULL, "suspend_lo", (size*(chunk/512)) * data);
	sysfs_set_num(sra, NULL, "sync_speed_min", speed);
	free(buf);
	return 1;
}

/*
 * If any spare contains md_back_data-1 which is recent wrt mtime,
 * write that data into the array and update the super blocks with
 * the new reshape_progress
 */
int Grow_restart(struct supertype *st, struct mdinfo *info, int *fdlist, int cnt,
		 char *backup_file, int verbose)
{
	int i, j;
	int old_disks;
	unsigned long long *offsets;
	unsigned long long  nstripe, ostripe;
	int ndata, odata;

	if (info->new_level != info->array.level)
		return 1; /* Cannot handle level changes (they are instantaneous) */

	odata = info->array.raid_disks - info->delta_disks - 1;
	if (info->array.level == 6) odata--; /* number of data disks */
	ndata = info->array.raid_disks - 1;
	if (info->new_level == 6) ndata--;

	old_disks = info->array.raid_disks - info->delta_disks;

	if (info->delta_disks <= 0)
		/* Didn't grow, so the backup file must have
		 * been used
		 */
		old_disks = cnt;
	for (i=old_disks-(backup_file?1:0); i<cnt; i++) {
		struct mdinfo dinfo;
		int fd;
		int bsbsize;
		char *devname, namebuf[20];

		/* This was a spare and may have some saved data on it.
		 * Load the superblock, find and load the
		 * backup_super_block.
		 * If either fail, go on to next device.
		 * If the backup contains no new info, just return
		 * else restore data and update all superblocks
		 */
		if (i == old_disks-1) {
			fd = open(backup_file, O_RDONLY);
			if (fd<0) {
				fprintf(stderr, Name ": backup file %s inaccessible: %s\n",
					backup_file, strerror(errno));
				continue;
			}
			devname = backup_file;
		} else {
			fd = fdlist[i];
			if (fd < 0)
				continue;
			if (st->ss->load_super(st, fd, NULL))
				continue;

			st->ss->getinfo_super(st, &dinfo, NULL);
			st->ss->free_super(st);

			if (lseek64(fd,
				    (dinfo.data_offset + dinfo.component_size - 8) <<9,
				    0) < 0) {
				fprintf(stderr, Name ": Cannot seek on device %d\n", i);
				continue; /* Cannot seek */
			}
			sprintf(namebuf, "device-%d", i);
			devname = namebuf;
		}
		if (read(fd, &bsb, sizeof(bsb)) != sizeof(bsb)) {
			if (verbose)
				fprintf(stderr, Name ": Cannot read from %s\n", devname);
			continue; /* Cannot read */
		}
		if (memcmp(bsb.magic, "md_backup_data-1", 16) != 0 &&
		    memcmp(bsb.magic, "md_backup_data-2", 16) != 0) {
			if (verbose)
				fprintf(stderr, Name ": No backup metadata on %s\n", devname);
			continue;
		}
		if (bsb.sb_csum != bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum)-((char*)&bsb))) {
			if (verbose)
				fprintf(stderr, Name ": Bad backup-metadata checksum on %s\n", devname);
			continue; /* bad checksum */
		}
		if (memcmp(bsb.magic, "md_backup_data-2", 16) == 0 &&
		    bsb.sb_csum2 != bsb_csum((char*)&bsb, ((char*)&bsb.sb_csum2)-((char*)&bsb))) {
			if (verbose)
				fprintf(stderr, Name ": Bad backup-metadata checksum2 on %s\n", devname);
			continue; /* Bad second checksum */
		}
		if (memcmp(bsb.set_uuid,info->uuid, 16) != 0) {
			if (verbose)
				fprintf(stderr, Name ": Wrong uuid on backup-metadata on %s\n", devname);
			continue; /* Wrong uuid */
		}

		/* array utime and backup-mtime should be updated at much the same time, but it seems that
		 * sometimes they aren't... So allow considerable flexability in matching, and allow
		 * this test to be overridden by an environment variable.
		 */
		if (info->array.utime > (int)__le64_to_cpu(bsb.mtime) + 2*60*60 ||
		    info->array.utime < (int)__le64_to_cpu(bsb.mtime) - 10*60) {
			if (check_env("MDADM_GROW_ALLOW_OLD")) {
				fprintf(stderr, Name ": accepting backup with timestamp %lu "
					"for array with timestamp %lu\n",
					(unsigned long)__le64_to_cpu(bsb.mtime),
					(unsigned long)info->array.utime);
			} else {
				if (verbose)
					fprintf(stderr, Name ": too-old timestamp on "
						"backup-metadata on %s\n", devname);
				continue; /* time stamp is too bad */
			}
		}

		if (bsb.magic[15] == '1') {
		if (info->delta_disks >= 0) {
			/* reshape_progress is increasing */
			if (__le64_to_cpu(bsb.arraystart) + __le64_to_cpu(bsb.length) <
			    info->reshape_progress) {
			nonew:
				if (verbose)
					fprintf(stderr, Name ": backup-metadata found on %s but is not needed\n", devname);
				continue; /* No new data here */
			}
		} else {
			/* reshape_progress is decreasing */
			if (__le64_to_cpu(bsb.arraystart) >=
			    info->reshape_progress)
				goto nonew; /* No new data here */
		}
		} else {
		if (info->delta_disks >= 0) {
			/* reshape_progress is increasing */
			if (__le64_to_cpu(bsb.arraystart) + __le64_to_cpu(bsb.length) <
			    info->reshape_progress &&
			    __le64_to_cpu(bsb.arraystart2) + __le64_to_cpu(bsb.length2) <
			    info->reshape_progress)
				goto nonew; /* No new data here */
		} else {
			/* reshape_progress is decreasing */
			if (__le64_to_cpu(bsb.arraystart) >=
			    info->reshape_progress &&
			    __le64_to_cpu(bsb.arraystart2) >=
			    info->reshape_progress)
				goto nonew; /* No new data here */
		}
		}
		if (lseek64(fd, __le64_to_cpu(bsb.devstart)*512, 0)< 0) {
		second_fail:
			if (verbose)
				fprintf(stderr, Name ": Failed to verify secondary backup-metadata block on %s\n",
					devname);
			continue; /* Cannot seek */
		}
		/* There should be a duplicate backup superblock 4k before here */
		if (lseek64(fd, -4096, 1) < 0 ||
		    read(fd, &bsb2, sizeof(bsb2)) != sizeof(bsb2))
			goto second_fail; /* Cannot find leading superblock */
		if (bsb.magic[15] == '1')
			bsbsize = offsetof(struct mdp_backup_super, pad1);
		else
			bsbsize = offsetof(struct mdp_backup_super, pad);
		if (memcmp(&bsb2, &bsb, bsbsize) != 0)
			goto second_fail; /* Cannot find leading superblock */

		/* Now need the data offsets for all devices. */
		offsets = malloc(sizeof(*offsets)*info->array.raid_disks);
		for(j=0; j<info->array.raid_disks; j++) {
			if (fdlist[j] < 0)
				continue;
			if (st->ss->load_super(st, fdlist[j], NULL))
				/* FIXME should be this be an error */
				continue;
			st->ss->getinfo_super(st, &dinfo, NULL);
			st->ss->free_super(st);
			offsets[j] = dinfo.data_offset * 512;
		}
		printf(Name ": restoring critical section\n");

		if (restore_stripes(fdlist, offsets,
				    info->array.raid_disks,
				    info->new_chunk,
				    info->new_level,
				    info->new_layout,
				    fd, __le64_to_cpu(bsb.devstart)*512,
				    __le64_to_cpu(bsb.arraystart)*512,
				    __le64_to_cpu(bsb.length)*512)) {
			/* didn't succeed, so giveup */
			if (verbose)
				fprintf(stderr, Name ": Error restoring backup from %s\n",
					devname);
			return 1;
		}
		
		if (bsb.magic[15] == '2' &&
		    restore_stripes(fdlist, offsets,
				    info->array.raid_disks,
				    info->new_chunk,
				    info->new_level,
				    info->new_layout,
				    fd, __le64_to_cpu(bsb.devstart)*512 +
				    __le64_to_cpu(bsb.devstart2)*512,
				    __le64_to_cpu(bsb.arraystart2)*512,
				    __le64_to_cpu(bsb.length2)*512)) {
			/* didn't succeed, so giveup */
			if (verbose)
				fprintf(stderr, Name ": Error restoring second backup from %s\n",
					devname);
			return 1;
		}


		/* Ok, so the data is restored. Let's update those superblocks. */

		if (info->delta_disks >= 0) {
			info->reshape_progress = __le64_to_cpu(bsb.arraystart) +
				__le64_to_cpu(bsb.length);
			if (bsb.magic[15] == '2') {
				unsigned long long p2 = __le64_to_cpu(bsb.arraystart2) +
					__le64_to_cpu(bsb.length2);
				if (p2 > info->reshape_progress)
					info->reshape_progress = p2;
			}
		} else {
			info->reshape_progress = __le64_to_cpu(bsb.arraystart);
			if (bsb.magic[15] == '2') {
				unsigned long long p2 = __le64_to_cpu(bsb.arraystart2);
				if (p2 < info->reshape_progress)
					info->reshape_progress = p2;
			}
		}
		for (j=0; j<info->array.raid_disks; j++) {
			if (fdlist[j] < 0) continue;
			if (st->ss->load_super(st, fdlist[j], NULL))
				continue;
			st->ss->getinfo_super(st, &dinfo, NULL);
			dinfo.reshape_progress = info->reshape_progress;
			st->ss->update_super(st, &dinfo,
					     "_reshape_progress",
					     NULL,0, 0, NULL);
			st->ss->store_super(st, fdlist[j]);
			st->ss->free_super(st);
		}
		return 0;
	}
	/* Didn't find any backup data, try to see if any
	 * was needed.
	 */
	if (info->delta_disks < 0) {
		/* When shrinking, the critical section is at the end.
		 * So see if we are before the critical section.
		 */
		unsigned long long first_block;
		nstripe = ostripe = 0;
		first_block = 0;
		while (ostripe >= nstripe) {
			ostripe += info->array.chunk_size / 512;
			first_block = ostripe * odata;
			nstripe = first_block / ndata / (info->new_chunk/512) *
				(info->new_chunk/512);
		}

		if (info->reshape_progress >= first_block)
			return 0;
	}
	if (info->delta_disks > 0) {
		/* See if we are beyond the critical section. */
		unsigned long long last_block;
		nstripe = ostripe = 0;
		last_block = 0;
		while (nstripe >= ostripe) {
			nstripe += info->new_chunk / 512;
			last_block = nstripe * ndata;
			ostripe = last_block / odata / (info->array.chunk_size/512) *
				(info->array.chunk_size/512);
		}

		if (info->reshape_progress >= last_block)
			return 0;
	}
	/* needed to recover critical section! */
	if (verbose)
		fprintf(stderr, Name ": Failed to find backup of critical section\n");
	return 1;
}

int Grow_continue(int mdfd, struct supertype *st, struct mdinfo *info,
		  char *backup_file)
{
	/* Array is assembled and ready to be started, but
	 * monitoring is probably required.
	 * So:
	 *   - start read-only
	 *   - set upper bound for resync
	 *   - initialise the 'suspend' boundaries
	 *   - switch to read-write
	 *   - fork and continue monitoring
	 */
	int err;
	int backup_list[1];
	unsigned long long backup_offsets[1];
	int odisks, ndisks, ochunk, nchunk,odata,ndata;
	unsigned long a,b,blocks,stripes;
	int backup_fd;
	int *fds;
	unsigned long long *offsets;
	int d;
	struct mdinfo *sra, *sd;
	int rv;
	unsigned long cache;
	int done = 0;

	err = sysfs_set_str(info, NULL, "array_state", "readonly");
	if (err)
		return err;

	/* make sure reshape doesn't progress until we are ready */
	sysfs_set_str(info, NULL, "sync_max", "0");
	sysfs_set_str(info, NULL, "array_state", "active"); /* FIXME or clean */

	sra = sysfs_read(-1, devname2devnum(info->sys_name),
			 GET_COMPONENT|GET_DEVS|GET_OFFSET|GET_STATE|
			 GET_CACHE);
	if (!sra)
		return 1;

	/* ndisks is not growing, so raid_disks is old and +delta is new */
	odisks = info->array.raid_disks;
	ndisks = odisks + info->delta_disks;
	odata = odisks - 1;
	ndata = ndisks - 1;
	if (info->array.level == 6) {
		odata--;
		ndata--;
	}
	ochunk = info->array.chunk_size;
	nchunk = info->new_chunk;

	a = (ochunk/512) * odata;
	b = (nchunk/512) * ndata;
	/* Find GCD */
	while (a != b) {
		if (a < b)
			b -= a;
		if (b < a)
			a -= b;
	}
	/* LCM == product / GCD */
	blocks = (ochunk/512) * (nchunk/512) * odata * ndata / a;

	if (ndata == odata)
		while (blocks * 32 < sra->component_size &&
		       blocks < 16*1024*2)
			blocks *= 2;
	stripes = blocks / (info->array.chunk_size/512) / odata;

	/* check that the internal stripe cache is
	 * large enough, or it won't work.
	 */
	cache = (nchunk < ochunk) ? ochunk : nchunk;
	cache = cache * 4 / 4096;
	if (cache < blocks / 8 / odisks + 16)
		/* Make it big enough to hold 'blocks' */
		cache = blocks / 8 / odisks + 16;
	if (sra->cache_size < cache)
		sysfs_set_num(sra, NULL, "stripe_cache_size",
			      cache+1);

	memset(&bsb, 0, 512);
	memcpy(bsb.magic, "md_backup_data-1", 16);
	memcpy(&bsb.set_uuid, info->uuid, 16);
	bsb.mtime = __cpu_to_le64(time(0));
	bsb.devstart2 = blocks;

	backup_fd = open(backup_file, O_RDWR|O_CREAT, S_IRUSR | S_IWUSR);
	if (backup_fd < 0) {
		fprintf(stderr, Name ": Cannot open backup file %s\n",
			backup_file ?: "- no backup-file given");
		return 1;
	}
	backup_list[0] = backup_fd;
	backup_offsets[0] = 8 * 512;
	fds = malloc(odisks * sizeof(fds[0]));
	offsets = malloc(odisks * sizeof(offsets[0]));
	for (d=0; d<odisks; d++)
		fds[d] = -1;

	for (sd = sra->devs; sd; sd = sd->next) {
		if (sd->disk.state & (1<<MD_DISK_FAULTY))
			continue;
		if (sd->disk.state & (1<<MD_DISK_SYNC)) {
			char *dn = map_dev(sd->disk.major,
					   sd->disk.minor, 1);
			fds[sd->disk.raid_disk]
				= dev_open(dn, O_RDONLY);
			offsets[sd->disk.raid_disk] = sd->data_offset*512;
			if (fds[sd->disk.raid_disk] < 0) {
				fprintf(stderr, Name ": %s: cannot open component %s\n",
					info->sys_name, dn?dn:"-unknown-");
				rv = 1;
				goto release;
			}
			free(dn);
		}
	}

	switch(fork()) {
	case 0:
		close(mdfd);
		mlockall(MCL_FUTURE);
		if (info->delta_disks < 0)
			done = child_shrink(-1, info, stripes,
					    fds, offsets,
					    info->array.raid_disks,
					    info->array.chunk_size,
					    info->array.level, info->array.layout,
					    odata,
					    1, backup_list, backup_offsets);
		else if (info->delta_disks == 0) {
			/* The 'start' is a per-device stripe number.
			 * reshape_progress is a per-array sector number.
			 * So divide by ndata * chunk_size
			 */
			unsigned long long start = info->reshape_progress / ndata;
			start /= (info->array.chunk_size/512);
			done = child_same_size(-1, info, stripes,
					       fds, offsets,
					       start,
					       info->array.raid_disks,
					       info->array.chunk_size,
					       info->array.level, info->array.layout,
					       odata,
					       1, backup_list, backup_offsets);
		}
		if (backup_file && done)
			unlink(backup_file);
		/* FIXME should I intuit a level change */
		exit(0);
	case -1:
		fprintf(stderr, Name ": Cannot run child to continue monitoring reshape: %s\n",
			strerror(errno));
		return 1;
	default:
		break;
	}
release:
	return 0;
}