Support "--build"ing arrays with bitmaps.
hot-removal of bitmaps
--re-add of drives recently removed.
assorted extra tests
Signed-off-by: Neil Brown <neilb@cse.unsw.edu.au>
int subdevs = 0, missing_disks = 0;
mddev_dev_t dv;
int bitmap_fd;
-/* unsigned long long size = ~0ULL; / * needed for bitmap only */
+ unsigned long long size = ~0ULL;
/* scan all devices, make sure they really are block devices */
for (dv = devlist; dv; dv=dv->next) {
}
/* now add the devices */
for ((i=0), (dv = devlist) ; dv ; i++, dv=dv->next) {
+ unsigned long dsize;
+ int fd;
if (strcmp("missing", dv->devname) == 0)
continue;
if (stat(dv->devname, &stb)) {
dv->devname);
goto abort;
}
+ fd = open(dv->devname, O_RDONLY|O_EXCL);
+ if (fd < 0) {
+ fprintf(stderr, Name ": Cannot open %s: %s\n",
+ dv->devname, strerror(errno));
+ goto abort;
+ }
+ if (ioctl(fd, BLKGETSIZE, &dsize) == 0 && dsize > 0) {
+ unsigned long long ldsize = dsize;
+ ldsize <<= 9;
+ if (size== 0 || ldsize < size)
+ size = ldsize;
+ }
+ close(fd);
if (vers>= 9000) {
mdu_disk_info_t disk;
disk.number = i;
return 1;
}
if (CreateBitmap(bitmap_file, 1, NULL, bitmap_chunk,
- delay, write_behind, 0/* FIXME size */)) {
+ delay, write_behind, size>>9)) {
return 1;
}
bitmap_fd = open(bitmap_file, O_RDWR);
return 1;
}
if (bmf.pathname[0]) {
+ if (strcmp(file,"none")==0) {
+ if (ioctl(fd, SET_BITMAP_FILE, -1)!= 0) {
+ fprintf(stderr, Name ": failed to remove bitmap %s\n",
+ bmf.pathname);
+ return 1;
+ }
+ return 0;
+ }
fprintf(stderr, Name ": %s already has a bitmap (%s)\n",
devname, bmf.pathname);
return 1;
return 1;
}
if (array.state & (1<<MD_SB_BITMAP_PRESENT)) {
+ if (strcmp(file, "none")==0) {
+ array.state &= ~(1<<MD_SB_BITMAP_PRESENT);
+ if (ioctl(fd, SET_ARRAY_INFO, &array)!= 0) {
+ fprintf(stderr, Name ": failed to remove internal bitmap.\n");
+ return 1;
+ }
+ return 0;
+ }
fprintf(stderr, Name ": Internal bitmap already present on %s\n",
devname);
return 1;
array.major_version, array.minor_version);
return 1;
}
- if (strcmp(file, "internal") == 0) {
+ if (strcmp(file, "none") == 0) {
+ fprintf(stderr, Name ": no bitmap found on %s\n", devname);
+ return 1;
+ } else if (strcmp(file, "internal") == 0) {
int d;
for (d=0; d< st->max_devs; d++) {
mdu_disk_info_t disk;
fprintf(stderr, Name ": failed to set internal bitmap.\n");
return 1;
}
- } else
- abort(); /* FIXME */
+ } else {
+ int uuid[4];
+ int bitmap_fd;
+ int d;
+ int max_devs = st->max_devs;
+ void *super = NULL;
+ if (chunk == UnSet)
+ chunk = DEFAULT_BITMAP_CHUNK;
+
+ /* try to load a superblock */
+ for (d=0; d<max_devs; d++) {
+ mdu_disk_info_t disk;
+ char *dv;
+ int fd2;
+ disk.number = d;
+ if (ioctl(fd, GET_DISK_INFO, &disk) < 0)
+ continue;
+ if ((disk.major==0 && disk.minor==0) ||
+ (disk.state & (1<<MD_DISK_REMOVED)))
+ continue;
+ dv = map_dev(disk.major, disk.minor);
+ if (!dv) continue;
+ fd2 = open(dv, O_RDONLY);
+ if (fd2 >= 0 &&
+ st->ss->load_super(st, fd2, &super, NULL) == 0) {
+ close(fd2);
+ st->ss->uuid_from_super(uuid, super);
+ break;
+ }
+ close(fd2);
+ }
+ if (d == max_devs) {
+ fprintf(stderr, Name ": cannot find UUID for array!\n");
+ return 1;
+ }
+ if (CreateBitmap(file, 0, (char*)uuid, chunk,
+ delay, write_behind, array.size*2ULL)) {
+ return 1;
+ }
+ bitmap_fd = open(file, O_RDWR);
+ if (bitmap_fd < 0) {
+ fprintf(stderr, Name ": weird: %s cannot be openned\n",
+ file);
+ return 1;
+ }
+ if (ioctl(fd, SET_BITMAP_FILE, bitmap_fd) < 0) {
+ fprintf(stderr, Name ": Cannot set bitmap file for %s: %s\n",
+ devname, strerror(errno));
+ return 1;
+ }
+ }
return 0;
}
int tfd;
struct supertype *st;
void *dsuper = NULL;
+ void *osuper = NULL; /* original super */
+ int duuid[4];
+ int ouuid[4];
if (ioctl(fd, GET_ARRAY_INFO, &array)) {
fprintf(stderr, Name ": cannot get array info for %s\n",
return 1;
case 'a':
/* add the device - hot or cold */
+ st = super_by_version(array.major_version,
+ array.minor_version);
+ if (!st) {
+ fprintf(stderr, Name ": unsupport array - version %d.%d\n",
+ array.major_version, array.minor_version);
+ return 1;
+ }
+
/* Make sure it isn't in use (in 2.6 or later) */
tfd = open(dv->devname, O_RDONLY|O_EXCL);
if (tfd < 0) {
dv->devname, strerror(errno));
return 1;
}
+ if (array.not_persistent==0)
+ st->ss->load_super(st, tfd, &osuper, NULL);
+ /* will use osuper later */
close(tfd);
+
if (array.major_version == 0 &&
md_get_version(fd)%100 < 2) {
if (ioctl(fd, HOT_ADD_DISK,
return 1;
}
- /* need to find a sample superblock to copy, and
- * a spare slot to use
- */
- st = super_by_version(array.major_version,
- array.minor_version);
- if (!st) {
- fprintf(stderr, Name ": unsupport array - version %d.%d\n",
- array.major_version, array.minor_version);
- return 1;
- }
- for (j=0; j<st->max_devs; j++) {
- char *dev;
- int dfd;
- disc.number = j;
- if (ioctl(fd, GET_DISK_INFO, &disc))
- continue;
- if (disc.major==0 && disc.minor==0)
- continue;
- if ((disc.state & 4)==0) continue; /* sync */
- /* Looks like a good device to try */
- dev = map_dev(disc.major, disc.minor);
- if (!dev) continue;
- dfd = open(dev, O_RDONLY);
- if (dfd < 0) continue;
- if (st->ss->load_super(st, dfd, &dsuper, NULL)) {
+ if (array.not_persistent == 0) {
+
+ /* need to find a sample superblock to copy, and
+ * a spare slot to use
+ */
+ for (j=0; j<st->max_devs; j++) {
+ char *dev;
+ int dfd;
+ disc.number = j;
+ if (ioctl(fd, GET_DISK_INFO, &disc))
+ continue;
+ if (disc.major==0 && disc.minor==0)
+ continue;
+ if ((disc.state & 4)==0) continue; /* sync */
+ /* Looks like a good device to try */
+ dev = map_dev(disc.major, disc.minor);
+ if (!dev) continue;
+ dfd = open(dev, O_RDONLY);
+ if (dfd < 0) continue;
+ if (st->ss->load_super(st, dfd, &dsuper, NULL)) {
+ close(dfd);
+ continue;
+ }
close(dfd);
- continue;
+ break;
+ }
+ if (!dsuper) {
+ fprintf(stderr, Name ": cannot find valid superblock in this array - HELP\n");
+ return 1;
+ }
+ /* Possibly this device was recently part of the array
+ * and was temporarily removed, and is now being re-added.
+ * If so, we can simply re-add it.
+ */
+ st->ss->uuid_from_super(duuid, dsuper);
+
+ if (osuper) {
+ st->ss->uuid_from_super(ouuid, osuper);
+ if (memcmp(duuid, ouuid, sizeof(ouuid))==0) {
+ /* look close enough for now. Kernel
+ * will worry about where a bitmap
+ * based reconstruct is possible
+ */
+ struct mdinfo mdi;
+ struct mddev_ident_s ident;
+ st->ss->getinfo_super(&mdi, &ident, osuper);
+ disc.major = major(stb.st_rdev);
+ disc.minor = minor(stb.st_rdev);
+ disc.number = mdi.disk.number;
+ disc.raid_disk = mdi.disk.raid_disk;
+ disc.state = mdi.disk.state;
+ if (ioctl(fd, ADD_NEW_DISK, &disc) == 0) {
+ if (verbose >= 0)
+ fprintf(stderr, Name ": re-added %s\n", dv->devname);
+ return 0;
+ }
+ /* fall back on normal-add */
+ }
}
- close(dfd);
- break;
- }
- if (!dsuper) {
- fprintf(stderr, Name ": cannot find valid superblock in this array - HELP\n");
- return 1;
}
for (j=0; j< st->max_devs; j++) {
disc.number = j;
disc.minor = minor(stb.st_rdev);
disc.number =j;
disc.state = 0;
- if (dv->writemostly)
- disc.state |= 1 << MD_DISK_WRITEMOSTLY;
- st->ss->add_to_super(dsuper, &disc);
- if (st->ss->write_init_super(st, dsuper, &disc, dv->devname))
- return 1;
+ if (array.not_persistent==0) {
+ if (dv->writemostly)
+ disc.state |= 1 << MD_DISK_WRITEMOSTLY;
+ st->ss->add_to_super(dsuper, &disc);
+ if (st->ss->write_init_super(st, dsuper, &disc, dv->devname))
+ return 1;
+ } else if (dv->re_add) {
+ /* this had better be raid1.
+ * As we are "--re-add"ing we must find a spare slot
+ * to fill.
+ */
+ char *used = malloc(array.raid_disks);
+ memset(used, 0, array.raid_disks);
+ for (j=0; j< st->max_devs; j++) {
+ mdu_disk_info_t disc2;
+ disc2.number = j;
+ if (ioctl(fd, GET_DISK_INFO, &disc2))
+ continue;
+ if (disc2.major==0 && disc2.minor==0)
+ continue;
+ if (disc2.state & 8) /* removed */
+ continue;
+ if (disc2.raid_disk < 0)
+ continue;
+ if (disc2.raid_disk > array.raid_disks)
+ continue;
+ used[disc2.raid_disk] = 1;
+ }
+ for (j=0 ; j<array.raid_disks; j++)
+ if (!used[j]) {
+ disc.raid_disk = j;
+ disc.state |= (1<<MD_DISK_SYNC);
+ break;
+ }
+ }
if (ioctl(fd,ADD_NEW_DISK, &disc)) {
fprintf(stderr, Name ": add new device failed for %s as %d: %s\n",
dv->devname, j, strerror(errno));
{"bitmap-chunk", 1, 0, 4},
{"write-behind", 2, 0, 5},
{"write-mostly",0, 0, 'W'},
+ {"re-add", 0, 0, 6},
/* For assemble */
{"uuid", 1, 0, 'u'},
.TP
.BR -b ", " --bitmap=
Give the name of a bitmap file to use with this array. Can be used
-with --create (file should not exist) or --assemble (file should
-exist).
+with --create (file should not exist), --assemble (file should
+exist), of --grow (file should not exist).
+
+The file
+.B internal
+can be used to indicate that the bitmap should be stored in the array,
+near the superblock. There is a limited amount of space for such
+bitmaps, but it is often sufficient.
+
+The file
+.B none
+can be given when used with --grow to remove a bitmap.
.TP
.BR --bitmap-chunk=
'''add, or
hotadd listed devices.
+.TP
+.BR --re-add
+Listed devices are assumed to have recently been part of the array,
+and they are re-added. This is only different from --add when a
+write-intent bitmap is present. It causes only those parts of the
+device that have changed since the device was removed from the array
+to be reconstructed.
+
+This flag is only needed with arrays that are built without a
+superblock (i.e. --build, not --create). For array with a superblock,
+.I mdadm
+checks if a superblock is present and automatically determines if a
+re-add is appropriate.
+
.TP
.BR -r ", " --remove
remove listed devices. They must not be active. i.e. they should
add a write-intent bitmap to a RAID1 array.
.PP
-Normally when an array is build the "size" it taken from the smallest
+Normally when an array is built the "size" it taken from the smallest
of the drives. If all the small drives in an arrays are, one at a
time, removed and replaced with larger drives, then you could have an
array of large drives with only a small amount used. In this
When the number of devices is increased, any hot spares that are
present may be activated immediately.
+A write-intent bitmap can be added to, or remove from, an active RAID1
+array. Either internal bitmap, of bitmaps stored in a separate file
+can be added. Note that if you add a bitmap stored in a file which is
+in a filesystem that is on the raid array being affected, the system
+will deadlock. The bitmap must be on a separate filesystem.
+
.SH EXAMPLES
.B " mdadm --query /dev/name-of-device"
int oneshot = 0;
struct supertype *ss = NULL;
int writemostly = 0;
+ int re_add = 0;
int copies;
case 'a':
case 'r':
case 'f':
+ case 6: /* re-add */
if (!mode) newmode = MANAGE;
break;
dv->devname = optarg;
dv->disposition = devmode;
dv->writemostly = writemostly;
+ dv->re_add = re_add;
dv->next = NULL;
*devlistend = dv;
devlistend = &dv->next;
dv->devname = optarg;
dv->disposition = devmode;
dv->writemostly = writemostly;
+ dv->re_add = re_add;
dv->next = NULL;
*devlistend = dv;
devlistend = &dv->next;
case O(GROW,'a'):
case O(MANAGE,'a'): /* add a drive */
devmode = 'a';
+ re_add = 0;
+ continue;
+ case O(MANAGE,6):
+ devmode = 'a';
+ re_add = 1;
continue;
case O(MANAGE,'r'): /* remove a drive */
devmode = 'r';
rv |= 1;
break;
}
- bitmap_fd = open(bitmap_file, O_RDWR,0);
- if (bitmap_fd < 0 && errno != ENOENT) {
- perror(Name ": cannot create bitmap file");
- rv |= 1;
- break;
- }
- if (bitmap_fd < 0) {
- bitmap_fd = CreateBitmap(bitmap_file, force, NULL,
- bitmap_chunk, delay, write_behind, size);
- }
}
rv = Build(devlist->devname, mdfd, chunk, level, layout,
raiddisks, devlist->next, assume_clean,
* Not set for names read from .config
*/
char writemostly;
+ char re_add;
struct mddev_dev_s *next;
} *mddev_dev_t;
echo >&2 "test: $mdadm isn't usable."
fi
-export check="sh $dir/tests/check"
-
# assume md0, md1, md2 exist in /dev
md0=/dev/md0 md1=/dev/md1 md2=/dev/md2
[ -f $targetdir/mdtest$d ] || dd if=/dev/zero of=$targetdir/mdtest$d count=$size bs=1K > /dev/null 2>&1
losetup /dev/loop$d $targetdir/mdtest$d
eval dev$d=/dev/loop$d
+ eval file$d=$targetdir/mdtest$d
eval devlist=\"\$devlist \$dev$d\"
done
echo >&2 "ERROR state $2 not found!"; cat /proc/mdstat ; exit 1; }
sleep 0.5
;;
+
+ bitmap )
+ grep -s bitmap > /dev/null /proc/mdstat || {
+ echo >&2 ERROR no bitmap ; cat /proc/mdstat ; exist 1; }
+ ;;
+ nobitmap )
+ if grep -s "bitmap" > /dev/null /proc/mdstat
+ then
+ echo >&2 ERROR bitmap present ; cat /proc/mdstat ; exit 1;
+ fi
+ ;;
* ) echo >&2 ERROR unknown check $1 ; exit 1;
esac
--- /dev/null
+
+# create a v-1 raid5 array and assemble in various ways
+
+mdadm -CR -e1 --name one $md1 -l5 -n3 $dev0 $dev1 $dev2
+tst="check raid5 ;testdev $md1 2 $mdsize1 64 ; mdadm -S $md1"
+uuid=`mdadm -Db $md1 | sed 's/.*UUID=//'`
+check wait
+
+eval $tst
+
+mdadm -A $md1 $dev0 $dev1 $dev2
+eval $tst
+
+mdadm -A $md1 -u $uuid $devlist
+eval $tst
+
+mdadm -A $md1 --name one $devlist
+eval $tst
+
+
+conf=$targetdir/mdadm.conf
+{
+ echo DEVICE $devlist
+ echo array $md1 UUID=$uuid
+} > $conf
+
+mdadm -As -c $conf $md1
+eval $tst
+
+{
+ echo DEVICE $devlist
+ echo array $md1 name=one
+} > $conf
+
+mdadm -As -c $conf
+eval $tst
+
+{
+ echo DEVICE $devlist
+ echo array $md1 devices=$dev0,$dev1,$dev2
+} > $conf
+
+mdadm -As -c $conf
+
+echo "DEVICE $devlist" > $conf
+mdadm -Db $md1 >> $conf
+eval $tst
+
+mdadm --assemble --scan --config=$conf $md1
+eval $tst
+
+echo " metadata=1.0 devices=$dev0,$dev1,$dev2" >> $conf
+mdadm --assemble --scan --config=$conf $md1
+eval $tst
+
+### Now with a missing device
+
+mdadm -AR $md1 $dev0 $dev2 #
+check state U_U
+eval $tst
+
+mdadm -A $md1 -u $uuid $devlist
+check state U_U
+eval $tst
+
+mdadm -A $md1 --name=one $devlist
+check state U_U
+eval $tst
+
+
+conf=$targetdir/mdadm.conf
+{
+ echo DEVICE $devlist
+ echo array $md1 UUID=$uuid
+} > $conf
+
+mdadm -As -c $conf $md1
+check state U_U
+eval $tst
+
+{
+ echo DEVICE $devlist
+ echo array $md1 name=one
+} > $conf
+
+mdadm -As -c $conf
+check state U_U
+eval $tst
+
+{
+ echo DEVICE $devlist
+ echo array $md1 devices=$dev0,$dev1,$dev2
+} > $conf
+
+mdadm -As -c $conf
+
+echo "DEVICE $devlist" > $conf
+mdadm -Db $md1 >> $conf
+check state U_U
+eval $tst
+
+mdadm --assemble --scan --config=$conf $md1
+check state U_U
+eval $tst
+
+echo " metadata=1.0 devices=$dev0,$dev1,$dev2" >> $conf
+mdadm --assemble --scan --config=$conf $md1
+check state U_U
+eval $tst
bmf=$targetdir/bm
rm -f $bmf
-mdadm -E $dev1
+#mdadm -E $dev1
mdadm --grow $md0 --bitmap=$bmf --delay=1 || { $mdadm -X $bmf ; exit 1; }
dirty1=`mdadm -X $bmf | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
sleep 4
sleep 4
dirty4=`mdadm -X $bmf | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
-echo $dirty1 $dirty2 $dirty3 $dirty4
+#echo $dirty1 $dirty2 $dirty3 $dirty4
if [ $dirty2 -ne 0 -o $dirty4 -ne 0 -o $dirty3 -lt 400 ]
then
echo bad dirty counts
exit 1
fi
+
+# now to remove the bitmap
+check bitmap
+mdadm --grow $md0 --bitmap=none
+check nobitmap
mdadm -S $md0
--- /dev/null
+
+#
+# create a raid1 array, add an internal bitmap
+#
+mdadm --create --run $md0 -l 1 -n 2 $dev1 $dev2
+check wait
+testdev $md0 1 $mdsize0 1
+
+#mdadm -E $dev1
+mdadm --grow $md0 --bitmap=internal --delay=1 || $mdadm -X $dev2
+dirty1=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+sleep 4
+dirty2=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+
+testdev $md0 1 $mdsize0 1
+dirty3=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+sleep 4
+dirty4=`mdadm -X $dev2 | sed -n -e 's/.*Bitmap.* \([0-9]*\) dirty.*/\1/p'`
+
+#echo $dirty1 $dirty2 $dirty3 $dirty4
+if [ $dirty2 -ne 0 -o $dirty4 -ne 0 -o $dirty3 -lt 400 ]
+then
+ echo bad dirty counts
+ exit 1
+fi
+
+# now to remove the bitmap
+check bitmap
+mdadm --grow $md0 --bitmap=none
+check nobitmap
+mdadm -S $md0
--- /dev/null
+
+#
+# create a raid1, remove a drive, and readd it.
+# resync should be instant.
+# Then do some IO first. Resync should still be very fast
+#
+
+mdadm -CR $md0 -l1 -n2 -binternal -d1 $dev1 $dev2
+check resync
+check wait
+testdev $md0 1 $mdsize0 1
+sleep 4
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+mdadm $md0 -a $dev2
+ cat /proc/mdstat
+check nosync
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+testdev $md0 1 $mdsize0 1
+mdadm $md0 -a $dev2
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+
+mdadm $md0 -f $dev2; sleep 1
+mdadm $md0 -r $dev2
+if dd if=/dev/zero of=$md0 ; then : ; fi
+mdadm $md0 -a $dev2
+check recovery
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+mdadm -S $md0
--- /dev/null
+
+#
+# create a raid1, remove a drive, and readd it.
+# resync should be instant.
+# Then do some IO first. Resync should still be very fast
+#
+bmf=$targetdir/bitmap2
+rm -f $bmf
+mdadm -B $md0 -l1 -n2 -b$bmf -d1 $dev1 $dev2
+check resync
+check wait
+testdev $md0 1 $size 1
+sleep 4
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+mdadm $md0 --re-add $dev2
+check nosync
+
+mdadm $md0 -f $dev2
+sleep 1
+mdadm $md0 -r $dev2
+testdev $md0 1 $size 1
+mdadm $md0 --re-add $dev2
+check wait
+cmp --bytes=$[$mdsize0*1024] $dev1 $dev2
+
+mdadm $md0 -f $dev2; sleep 1
+mdadm $md0 -r $dev2
+if dd if=/dev/zero of=$md0 ; then : ; fi
+mdadm $md0 --re-add $dev2
+check recovery
+check wait
+# should BLKFLSBUF and then read $dev1/$dev2...
+cmp --bytes=$[$mdsize0*1024] $file1 $file2
+mdadm -S $md0
mdadm -S $md0
mdadm -A $md0 --name="Fred" $devlist
-mdadm -Db $md0
+#mdadm -Db $md0
mdadm -S $md0
mdadm --assemble --update=byteorder $md0 $dev0 $dev1 $dev2 $dev3
sleep 3
-cat /proc/mdstat
+check recovery
mdadm -S $md0
--- /dev/null
+exit 0
+mdadm -CR $md0 -l1 -n3 $dev1 $dev2 $dev3
+
+ls -Rl /sys/block/md0
+
+cat /sys/block/md0/md/level
+cat /sys/block/md0/md/raid_disks
+
+mdadm -S $md0
+
+exit 1
projects / thirdparty / mdadm.git / commitdiff
