test

   1 #!/bin/bash
   2 #
   3 # run test suite for mdadm
   4 dir=$(pwd)
   5 mdadm=$dir/mdadm
   6 testdir="tests"
   7 targetdir="/var/tmp"
   8 logdir="$targetdir"
   9 config=/tmp/mdadm.conf
  10
  11 savelogs=0
  12 exitonerror=1
  13 prefix='[0-9][0-9]'
  14
  15 # use loop devices by default if doesn't specify --dev
  16 DEVTYPE=loop
  17 INTEGRITY=yes
  18 LVM_VOLGROUP=mdtest
  19
  20 # make sure to test local mdmon, not system one
  21 export MDADM_NO_SYSTEMCTL=1
  22
  23 # assume md0, md1, md2 exist in /dev
  24 md0=/dev/md0
  25 md1=/dev/md1
  26 md2=/dev/md2
  27 mdp0=/dev/md_d0
  28 mdp1=/dev/md_d1
  29
  30 # We test mdadm on loop-back block devices.
  31 # dir for storing files should be settable by command line maybe
  32 size=20000
  33 # super0, round down to multiple of 64 and substract 64
  34 mdsize0=19904
  35 # super00 is nested, subtract 128
  36 mdsize00=19840
  37 # super1.0 round down to multiple of 2, subtract 8
  38 mdsize1=19992
  39 mdsize1a=19988
  40 mdsize12=19988
  41 # super1.2 for linear: round to multiple of 2, subtract 4
  42 mdsize1_l=19996
  43 mdsize2_l=19996
  44 # subtract another 4 for bitmaps
  45 mdsize1b=19988
  46 mdsize11=19992
  47 mdsize11a=19456
  48 mdsize12=19988
  49
  50 # ddf needs bigger devices as 32Meg is reserved!
  51 ddfsize=65536
  52
  53 # $1 is optional parameter, it shows why to save log
  54 save_log() {
  55         status=$1
  56         logfile="$status""$_basename".log
  57
  58         cat $targetdir/stderr >> $targetdir/log
  59         cp $targetdir/log $logdir/$_basename.log
  60         echo "## $HOSTNAME: saving dmesg." >> $logdir/$logfile
  61         dmesg -c >> $logdir/$logfile
  62         echo "## $HOSTNAME: saving proc mdstat." >> $logdir/$logfile
  63         cat /proc/mdstat >> $logdir/$logfile
  64         array=($(mdadm -Ds | cut -d' ' -f2))
  65         [ "$1" == "fail" ] &&
  66                 echo "FAILED - see $logdir/$_basename.log and $logdir/$logfile for details"
  67         if [ $DEVTYPE == 'lvm' ]
  68         then
  69                 # not supported lvm type yet
  70                 echo
  71         elif [ "$DEVTYPE" == 'loop' -o "$DEVTYPE" == 'disk' ]
  72         then
  73                 if [ ! -z "$array" -a ${#array[@]} -ge 1 ]
  74                 then
  75                         echo "## $HOSTNAME: mdadm -D ${array[@]}" >> $logdir/$logfile
  76                         $mdadm -D ${array[@]} >> $logdir/$logfile
  77                         # ignore saving external(external file, imsm...) bitmap
  78                         cat /proc/mdstat | grep -q "linear\|external" && return 0
  79                         md_disks=($($mdadm -D -Y ${array[@]} | grep "/dev/" | cut -d'=' -f2))
  80                         cat /proc/mdstat | grep -q "bitmap"
  81                         if [ $? -eq 0 ]
  82                         then
  83                                 echo "## $HOSTNAME: mdadm -X ${md_disks[@]}" >> $logdir/$logfile
  84                                 $mdadm -X ${md_disks[@]} >> $logdir/$logfile
  85                         fi
  86                 else
  87                         echo "## $HOSTNAME: no array assembled!" >> $logdir/$logfile
  88                 fi
  89         fi
  90 }
  91
  92 die() {
  93         echo -e "\n\tERROR: $* \n"
  94         save_log fail
  95         exit 2
  96 }
  97
  98 cleanup() {
  99         udevadm settle
 100         $mdadm -Ssq 2> /dev/null
 101         case $DEVTYPE in
 102         loop )
 103                 for d in 0 1 2 3 4 5 6 7  8 9 10 11 12 13
 104                 do
 105                         losetup -d /dev/loop$d
 106                         rm -f /dev/disk/by-path/loop*
 107                         rm -f /var/tmp/mdtest$d
 108                 done
 109                 ;;
 110         lvm )
 111                 for d in 0 1 2 3 4 5 6 7  8 9 10 11 12 13
 112                 do
 113                         eval "lvremove --quiet -f \$dev$d"
 114                 done
 115                 ;;
 116         disk )
 117                 $mdadm --zero ${disks[@]} &> /dev/null
 118                 ;;
 119         esac
 120 }
 121
 122 ctrl_c() {
 123         exitonerror=1
 124 }
 125
 126 do_setup() {
 127         trap cleanup 0 1 3 15
 128         trap ctrl_c 2
 129
 130         [ -d $logdir ] || mkdir -p $logdir
 131         dmesg -c > /dev/null
 132
 133         devlist=
 134         if [ "$DEVTYPE" == "loop" ]
 135         then
 136                 # make sure there are no loop devices remaining.
 137                 # udev started things can sometimes prevent them being stopped
 138                 # immediately
 139                 while grep loop /proc/partitions > /dev/null 2>&1
 140                 do
 141                         $mdadm -Ssq
 142                         losetup -d /dev/loop[0-9]* 2> /dev/null
 143                         sleep 0.2
 144                 done
 145         elif [ "$DEVTYPE" == "disk" ]
 146         then
 147                 if [ ! -z "$disks" ]
 148                 then
 149                         for d in $(seq 0 ${#disks[@]})
 150                         do
 151                                 eval "dev$d=${disks[$d]}"
 152                                 eval devlist=\"\$devlist \$dev$d\"
 153                                 eval devlist$d=\"\$devlist\"
 154                         done
 155                         $mdadm --zero ${disks[@]} &> /dev/null
 156                 else
 157                         echo "Forget to provide physical devices for disk mode."
 158                         exit 1
 159                 fi
 160         fi
 161         for d in 0 1 2 3 4 5 6 7 8 9 10 11 12 13
 162         do
 163                 sz=$size
 164                 [ $d -gt 7 ] && sz=$ddfsize
 165                 case $DEVTYPE in
 166                 loop)
 167                         [ -f $targetdir/mdtest$d ] ||
 168                                 dd if=/dev/zero of=$targetdir/mdtest$d count=$sz bs=1K > /dev/null 2>&1
 169                         # make sure udev doesn't touch
 170                         mdadm --zero $targetdir/mdtest$d 2> /dev/null
 171                         [ -b /dev/loop$d ] || mknod /dev/loop$d b 7 $d
 172                         if [ $d -eq 7 ]
 173                         then
 174                                 losetup /dev/loop$d $targetdir/mdtest6 # for multipath use
 175                         else
 176                                 losetup /dev/loop$d $targetdir/mdtest$d
 177                         fi
 178                         eval dev$d=/dev/loop$d
 179                         eval file$d=$targetdir/mdtest$d
 180                         ;;
 181                 lvm)
 182                         unset MULTIPATH
 183                         eval dev$d=/dev/mapper/${LVM_VOLGROUP}-mdtest$d
 184                         if ! lvcreate --quiet -L ${sz}K -n mdtest$d $LVM_VOLGROUP
 185                         then
 186                                 trap '' 0 # make sure lvremove is not called
 187                                 eval echo error creating \$dev$d
 188                                 exit 129
 189                         fi
 190                         ;;
 191                 ram)
 192                         unset MULTIPATH
 193                         eval dev$d=/dev/ram$d
 194                         ;;
 195                 esac
 196                 eval devlist=\"\$devlist \$dev$d\"
 197                 eval devlist$d=\"\$devlist\"
 198                 #" <-- add this quote to un-confuse vim syntax highlighting
 199         done
 200         path0=$dev6
 201         path1=$dev7
 202         ulimit -c unlimited
 203         [ -f /proc/mdstat ] || modprobe md_mod
 204         echo 2000 > /proc/sys/dev/raid/speed_limit_max
 205         echo 0 > /sys/module/md_mod/parameters/start_ro
 206 }
 207
 208 # mdadm always adds --quiet, and we want to see any unexpected messages
 209 mdadm() {
 210         rm -f $targetdir/stderr
 211         case $* in
 212         *-S* )
 213                 udevadm settle
 214                 p=`cat /proc/sys/dev/raid/speed_limit_max`
 215                 echo 20000 > /proc/sys/dev/raid/speed_limit_max
 216                 ;;
 217         esac
 218         case $* in
 219         *-C* | *--create* | *-B* | *--build* )
 220                 # clear superblock every time once creating or
 221                 # building arrays, because it's always creating
 222                 # and building array many times in a test case.
 223                 for args in $*
 224                 do
 225                         [[ $args =~ "/dev/" ]] && {
 226                                 [[ $args =~ "md" ]] ||
 227                                         $mdadm --zero $args > /dev/null
 228                         }
 229                 done
 230                 $mdadm 2> $targetdir/stderr --quiet "$@" --auto=yes
 231                 ;;
 232         * )
 233                 $mdadm 2> $targetdir/stderr --quiet "$@"
 234                 ;;
 235         esac
 236         rv=$?
 237         case $* in
 238         *-S* )
 239                 udevadm settle
 240                 echo $p > /proc/sys/dev/raid/speed_limit_max
 241                 ;;
 242         esac
 243         cat >&2 $targetdir/stderr
 244         return $rv
 245 }
 246
 247 # check various things
 248 check() {
 249         case $1 in
 250         spares )
 251                 spares=$(tr '] ' '\012\012' < /proc/mdstat | grep -c '(S)' || exit 0)
 252                 [ $spares -ne $2 ] &&
 253                         die "expected $2 spares, found $spares"
 254                 ;;
 255         raid* | linear )
 256                 grep -sq "active $1 " /proc/mdstat ||
 257                         die "active $1 not found"
 258                 ;;
 259         algorithm )
 260                 grep -sq " algorithm $2 " /proc/mdstat ||
 261                         die "algorithm $2 not found"
 262                 ;;
 263         resync | recovery | reshape )
 264                 cnt=5
 265                 while ! grep -sq $1 /proc/mdstat
 266                 do
 267                         if [ $cnt -gt 0 ] && grep -v idle /sys/block/md*/md/sync_action > /dev/null
 268                         then # Something isn't idle - wait a bit
 269                                 sleep 0.5
 270                                 cnt=$[cnt-1]
 271                         else
 272                                 die "no $1 happening"
 273                         fi
 274                 done
 275                 ;;
 276         nosync )
 277                 sleep 0.5
 278                 # Since 4.2 we delay the close of recovery until there has been a chance for
 279                 # spares to be activated.  That means that a recovery that finds nothing
 280                 # to do can still take a little longer than expected.
 281                 # add an extra check: is sync_completed shows the end is reached, assume
 282                 # there is no recovery.
 283                 if grep -sq -E '(resync|recovery|reshape) *=' /proc/mdstat
 284                 then
 285                         incomplete=`grep / /sys/block/md*/md/sync_completed 2> /dev/null | sed '/^ *\([0-9]*\) \/ \1/d'`
 286                         [ -n "$incomplete" ] &&
 287                                 die "resync or recovery is happening!"
 288                 fi
 289                 ;;
 290         wait )
 291                 p=`cat /proc/sys/dev/raid/speed_limit_max`
 292                 echo 2000000 > /proc/sys/dev/raid/speed_limit_max
 293                 sleep 0.1
 294                 while grep -Eq '(resync|recovery|reshape|check|repair) *=' /proc/mdstat ||
 295                         grep -v idle > /dev/null /sys/block/md*/md/sync_action
 296                 do
 297                         sleep 0.5
 298                 done
 299                 echo $p > /proc/sys/dev/raid/speed_limit_max
 300                 ;;
 301         state )
 302                 grep -sq "blocks.*\[$2\]\$" /proc/mdstat ||
 303                         die "state $2 not found!"
 304                 sleep 0.5
 305                 ;;
 306         bitmap )
 307                 grep -sq bitmap /proc/mdstat ||
 308                         die "no bitmap"
 309                 ;;
 310         nobitmap )
 311                 grep -sq "bitmap" /proc/mdstat &&
 312                         die "bitmap present"
 313                 ;;
 314         readonly )
 315                 grep -sq "read-only" /proc/mdstat ||
 316                         die "array is not read-only!"
 317                 ;;
 318         inactive )
 319                 grep -sq "inactive" /proc/mdstat ||
 320                         die "array is not inactive!"
 321                 ;;
 322         # It only can be used when there is only one raid
 323         chunk )
 324                 chunk_size=`awk -F',' '/chunk/{print $2}' /proc/mdstat | awk -F'[a-z]' '{print $1}'`
 325                 if [ "$chunk_size" -ne "$2" ] ; then
 326                         die "chunksize should be $2, but it's $chunk_size"
 327                 fi
 328                 ;;
 329         * )
 330                 die "unknown check $1"
 331                 ;;
 332         esac
 333 }
 334
 335 no_errors() {
 336         if [ -s $targetdir/stderr ]
 337         then
 338                 echo Bad errors from mdadm:
 339                 cat $targetdir/stderr
 340                 exit 2
 341         fi
 342 }
 343
 344 # basic device test
 345 testdev() {
 346         [ -b $1 ] || die "$1 isn't a block device."
 347         [ "$DEVTYPE" == "disk" ] && return 0
 348         udevadm settle
 349         dev=$1
 350         cnt=$2
 351         dvsize=$3
 352         chunk=$4
 353         if [ -z "$5" ]
 354         then
 355                 mkfs.ext3 -F -j $dev > /dev/null 2>&1 && fsck -fn $dev >&2
 356         fi
 357         dsize=$[dvsize/chunk]
 358         dsize=$[dsize*chunk]
 359         rasize=$[dsize*2*cnt]
 360         # rasize is in sectors
 361         if [ -n "$DEV_ROUND_K" ]
 362         then
 363                 rasize=$[rasize/DEV_ROUND_K/2]
 364                 rasize=$[rasize*DEV_ROUND_K*2]
 365         fi
 366         [ `/sbin/blockdev --getsize $dev` -eq 0 ] && sleep 2
 367         _sz=`/sbin/blockdev --getsize $dev`
 368         [ $rasize -lt $_sz -o $[rasize*4/5] -gt $_sz ] &&
 369                 die "size is wrong for $dev: $cnt * $dvsize (chunk=$chunk) = $rasize, not $_sz"
 370         return 0
 371 }
 372
 373 rotest() {
 374         dev=$1
 375         fsck -fn $dev >&2
 376 }
 377
 378 do_test() {
 379         _script=$1
 380         _basename=`basename $_script`
 381         if [ -f "$_script" ]
 382         then
 383                 rm -f $targetdir/stderr
 384                 # stop all arrays, just incase some script left an array active.
 385                 $mdadm -Ssq 2> /dev/null
 386                 mdadm --zero $devlist 2> /dev/null
 387                 # this might have been reset: restore the default.
 388                 echo 2000 > /proc/sys/dev/raid/speed_limit_max
 389                 # source script in a subshell, so it has access to our
 390                 # namespace, but cannot change it.
 391                 echo -ne "$_script... "
 392                 if ( set -ex ; . $_script ) &> $targetdir/log
 393                 then
 394                         dmesg | grep -iq "error\|call trace\|segfault" &&
 395                                 die "dmesg prints errors when testing $_basename!"
 396                         echo "succeeded"
 397                         _fail=0
 398                 else
 399                         save_log fail
 400                         _fail=1
 401                 fi
 402                 [ "$savelogs" == "1" ] &&
 403                         mv -f $targetdir/log $logdir/$_basename.log
 404                 [ "$_fail" == "1" -a "$exitonerror" == "1" ] && exit 1
 405         fi
 406 }
 407
 408 do_help() {
 409         cat <<-EOF
 410         Usage: $0 [options]
 411         Example for disk mode: ./test --dev=disk --disks=/dev/sda{2..15}
 412         Options:
 413                 --tests=test1,test2,...     Comma separated list of tests to run
 414                 --raidtype=                 raid0|linear|raid1|raid456|raid10|ddf|imsm
 415                 --disable-multipath         Disable any tests involving multipath
 416                 --disable-integrity         Disable slow tests of RAID[56] consistency
 417                 --logdir=directory          Directory to save all logfiles in
 418                 --save-logs                 Usually use with --logdir together
 419                 --keep-going | --no-error   Don't stop on error, ie. run all tests
 420                 --dev=loop|lvm|ram|disk     Use loop devices (default), LVM, RAM or disk
 421                 --disks=                    Provide a bunch of physical devices for test
 422                 --volgroup=name             LVM volume group for LVM test
 423                 setup                       Setup test environment and exit
 424                 cleanup                     Cleanup test environment
 425                 prefix                      Run tests with <prefix>
 426                 --help | -h                 Print this usage
 427         EOF
 428 }
 429
 430 parse_args() {
 431         for i in $*
 432         do
 433                 case $i in
 434                 [0-9][0-9] )
 435                         prefix=$i
 436                         ;;
 437                 setup )
 438                         echo "mdadm test environment setup"
 439                         do_setup
 440                         trap 0
 441                         exit 0
 442                         ;;
 443                 cleanup )
 444                         cleanup
 445                         exit 0
 446                         ;;
 447                 --tests=* )
 448                         TESTLIST=($(echo ${i##*=} | sed -e 's/,/ /g'))
 449                         ;;
 450                 --raidtype=* )
 451                         case ${i##*=} in
 452                         raid0 )
 453                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]r0\|raid0"))
 454                                 ;;
 455                         linear )
 456                                 TESTLIST=($(ls $testdir | grep "linear"))
 457                                 ;;
 458                         raid1 )
 459                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]r1\|raid1" | grep -vi raid10))
 460                                 ;;
 461                         raid456 )
 462                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]r[4-6]\|raid[4-6]"))
 463                                 ;;
 464                         raid10 )
 465                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]r10\|raid10"))
 466                                 ;;
 467                         ddf )
 468                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]ddf"))
 469                                 ;;
 470                         imsm )
 471                                 TESTLIST=($(ls $testdir | grep "[0-9][0-9]imsm"))
 472                                 ;;
 473                         * )
 474                                 echo "Unknown argument: $i"
 475                                 do_help
 476                                 exit 1
 477                                 ;;
 478                         esac
 479                         ;;
 480                 --logdir=* )
 481                         logdir="${i##*=}"
 482                         ;;
 483                 --save-logs )
 484                         savelogs=1
 485                         ;;
 486                 --keep-going | --no-error )
 487                         exitonerror=0
 488                         ;;
 489                 --disable-multipath )
 490                         unset MULTIPATH
 491                         ;;
 492                 --disable-integrity )
 493                         unset INTEGRITY
 494                         ;;
 495                 --dev=* )
 496                         case ${i##*=} in
 497                         loop )
 498                                 DEVTYPE=loop
 499                                 ;;
 500                         lvm )
 501                                 DEVTYPE=lvm
 502                                 ;;
 503                         ram )
 504                                 DEVTYPE=ram
 505                                 ;;
 506                         disk )
 507                                 DEVTYPE=disk
 508                                 ;;
 509                         * )
 510                                 echo "Unknown argument: $i"
 511                                 do_help
 512                                 exit 1
 513                                 ;;
 514                         esac
 515                         ;;
 516                 --disks=* )
 517                         disks=(${disks[*]} ${i##*=})
 518                         ;;
 519                 --volgroup=* )
 520                         LVM_VOLGROUP=`expr "x$i" : 'x[^=]*=\(.*\)'`
 521                         ;;
 522                 --help | -h )
 523                         do_help
 524                         exit 0
 525                         ;;
 526                 * )
 527                         echo " $0: Unknown argument: $i"
 528                         do_help
 529                         exit 1
 530                         ;;
 531                 esac
 532         done
 533 }
 534
 535 check_env() {
 536         user=$(id -un)
 537         [ "X$user" != "Xroot" ] && {
 538                 echo "test: testing can only be done as 'root'."
 539                 exit 1
 540         }
 541         [ -x "raid6check" -a -x $mdadm ] || {
 542                 echo "test: please run 'make everything' before perform testing."
 543                 exit 1
 544         }
 545         cmds=(mdadm lsblk df udevadm losetup mkfs.ext3 fsck seq)
 546         for cmd in ${cmds[@]}
 547         do
 548                 which $cmd > /dev/null || {
 549                         echo "$cmd command not found!"
 550                         exit 1
 551                 }
 552         done
 553         mdadm_src_ver="$($mdadm -V 2>&1)"
 554         mdadm_sbin_ver="$($(which mdadm) -V 2>&1)"
 555         if [ "$mdadm_src_ver" != "$mdadm_sbin_ver" ]
 556         then
 557                 # it's nessesary to 'make install' mdadm to /SBIN/DIR,
 558                 # such as systemd/mdadm-grow-continue@.service, would
 559                 # run as an instance by systemd when reshape happens,
 560                 # thus ensure that the correct mdadm is in testing.
 561                 echo "test: please run 'make install' before testing."
 562                 exit 1
 563         fi
 564         if ! $(df -T . | grep -iq ext)
 565         then
 566                 # 'external file' bitmap only supports with ext[2-4] file system
 567                 echo "test: please run test suite with ext[2-4] file system."
 568                 exit 1
 569         fi
 570         if $(lsblk -a | grep -iq raid)
 571         then
 572                 # donot run mdadm -Ss directly if there are RAIDs working.
 573                 echo "test: please run test suite without running RAIDs environment."
 574                 exit 1
 575         fi
 576         # Check whether to run multipath tests
 577         modprobe multipath 2> /dev/null
 578         grep -sq 'Personalities : .*multipath' /proc/mdstat &&
 579                 MULTIPATH="yes"
 580 }
 581
 582 main() {
 583         check_env
 584         do_setup
 585
 586         echo "Testing on linux-$(uname -r) kernel"
 587         [ "$savelogs" == "1" ] &&
 588                 echo "Saving logs to $logdir"
 589         if [ "x$TESTLIST" != "x" ]
 590         then
 591                 for script in ${TESTLIST[@]}
 592                 do
 593                         do_test $testdir/$script
 594                 done
 595         else
 596                 for script in $testdir/$prefix $testdir/$prefix*[^~]
 597                 do
 598                         do_test $script
 599                 done
 600         fi
 601
 602         exit 0
 603 }
 604
 605 parse_args $@
 606 main