clustermd_tests/func.sh

   1 #!/bin/bash
   2
   3 check_ssh()
   4 {
   5         NODE1="$(grep '^NODE1' $CLUSTER_CONF | cut -d'=' -f2)"
   6         NODE2="$(grep '^NODE2' $CLUSTER_CONF | cut -d'=' -f2)"
   7         [ -z "$NODE1" -o -z "$NODE2" ] && {
   8                 echo "Please provide node-ip in $CLUSTER_CONF."
   9                 exit 1
  10         }
  11         for ip in $NODE1 $NODE2
  12         do
  13                 ssh -o NumberOfPasswordPrompts=0 $ip -l root "pwd" > /dev/null
  14                 [ $? -ne 0 ] && {
  15                         echo "Please setup ssh-access with no-authorized mode."
  16                         exit 1
  17                 }
  18         done
  19 }
  20
  21 fetch_devlist()
  22 {
  23         ISCSI_ID="$(grep '^ISCSI_TARGET_ID' $CLUSTER_CONF | cut -d'=' -f2)"
  24         devlist="$(grep '^devlist' $CLUSTER_CONF | cut -d'=' -f2)"
  25         if [ ! -z "$ISCSI_ID" -a ! -z "$devlist" ]
  26         then
  27                 echo "Config ISCSI_TARGET_ID or devlist in $CLUSTER_CONF."
  28                 exit 1
  29         elif [ ! -z "$ISCSI_ID" -a -z "$devlist" ]
  30         then
  31                 for ip in $NODE1 $NODE2
  32                 do
  33                         ssh $ip "ls /dev/disk/by-path/*$ISCSI_ID*" > /dev/null
  34                         [ $? -ne 0 ] && {
  35                                 echo "$ip: No disks found in '$ISCSI_ID' connection."
  36                                 exit 1
  37                         }
  38                 done
  39                 devlist=($(ls /dev/disk/by-path/*$ISCSI_ID*))
  40         fi
  41         # sbd disk cannot use in testing
  42         for i in ${devlist[@]}
  43         do
  44                 sbd -d $i dump &> /dev/null
  45                 [ $? -eq '0' ] && devlist=(${devlist[@]#$i})
  46         done
  47         for i in $(seq 0 ${#devlist[@]})
  48         do
  49                 eval "dev$i=${devlist[$i]}"
  50         done
  51         [ "${#devlist[@]}" -lt 6 ] && {
  52                 echo "Cluster-md testing requires 6 disks at least."
  53                 exit 1
  54         }
  55 }
  56
  57 check_dlm()
  58 {
  59         if ! crm configure show | grep -q dlm
  60         then
  61                 crm configure primitive dlm ocf:pacemaker:controld \
  62                         op monitor interval=60 timeout=60 \
  63                         meta target-role=Started &> /dev/null
  64                 crm configure group base-group dlm
  65                 crm configure clone base-clone base-group \
  66                         meta interleave=true
  67         fi
  68         sleep 1
  69         for ip in $NODE1 $NODE2
  70         do
  71                 ssh $ip "pgrep dlm_controld > /dev/null" || {
  72                         echo "$ip: dlm_controld daemon doesn't exist."
  73                         exit 1
  74                 }
  75         done
  76         crm_mon -r -n1 | grep -iq "fail\|not" && {
  77                 echo "Please clear cluster-resource errors."
  78                 exit 1
  79         }
  80 }
  81
  82 check_env()
  83 {
  84         user=$(id -un)
  85         [ "X$user" = "Xroot" ] || {
  86                 echo "testing can only be done as 'root'."
  87                 exit 1
  88         }
  89         [ \! -x $mdadm ] && {
  90                 echo "test: please run make everything before perform testing."
  91                 exit 1
  92         }
  93         check_ssh
  94         commands=(mdadm iscsiadm bc modinfo dlm_controld
  95                   udevadm crm crm_mon lsblk pgrep sbd)
  96         for ip in $NODE1 $NODE2
  97         do
  98                 for cmd in ${commands[@]}
  99                 do
 100                         ssh $ip "which $cmd &> /dev/null" || {
 101                                 echo "$ip: $cmd, command not found!"
 102                                 exit 1
 103                         }
 104                 done
 105                 mods=(raid1 raid10 md_mod dlm md-cluster)
 106                 for mod in ${mods[@]}
 107                 do
 108                         ssh $ip "modinfo $mod > /dev/null" || {
 109                                 echo "$ip: $mod, module doesn't exist."
 110                                 exit 1
 111                         }
 112                 done
 113                 ssh $ip "lsblk -a | grep -iq raid"
 114                 [ $? -eq 0 ] && {
 115                         echo "$ip: Please run testing without running RAIDs environment."
 116                         exit 1
 117                 }
 118                 ssh $ip "modprobe md_mod"
 119         done
 120         fetch_devlist
 121         check_dlm
 122         [ -d $logdir ] || mkdir -p $logdir
 123 }
 124
 125 # $1/node, $2/optional
 126 stop_md()
 127 {
 128         if [ "$1" == "all" ]
 129         then
 130                 NODES=($NODE1 $NODE2)
 131         elif [ "$1" == "$NODE1" -o "$1" == "$NODE2" ]
 132         then
 133                 NODES=$1
 134         else
 135                 die "$1: unknown parameter."
 136         fi
 137         if [ -z "$2" ]
 138         then
 139                 for ip in ${NODES[@]}
 140                 do
 141                         ssh $ip mdadm -Ssq
 142                 done
 143         else
 144                 for ip in ${NODES[@]}
 145                 do
 146                         ssh $ip mdadm -S $2
 147                 done
 148         fi
 149 }
 150
 151 # $1/optional, it shows why to save log
 152 save_log()
 153 {
 154         status=$1
 155         logfile="$status""$_basename".log
 156
 157         cat $targetdir/stderr >> $targetdir/log
 158         cp $targetdir/log $logdir/$_basename.log
 159
 160         for ip in $NODE1 $NODE2
 161         do
 162                 echo "##$ip: saving dmesg." >> $logdir/$logfile
 163                 ssh $ip "dmesg -c" >> $logdir/$logfile
 164                 echo "##$ip: saving proc mdstat." >> $logdir/$logfile
 165                 ssh $ip "cat /proc/mdstat" >> $logdir/$logfile
 166                 array=($(ssh $ip "mdadm -Ds | cut -d' ' -f2"))
 167
 168                 if [ ! -z "$array" -a ${#array[@]} -ge 1 ]
 169                 then
 170                         echo "##$ip: mdadm -D ${array[@]}" >> $logdir/$logfile
 171                         ssh $ip "mdadm -D ${array[@]}" >> $logdir/$logfile
 172                         md_disks=($(ssh $ip "mdadm -DY ${array[@]} | grep "/dev/" | cut -d'=' -f2"))
 173                         cat /proc/mdstat | grep -q "bitmap"
 174                         if [ $? -eq 0 ]
 175                         then
 176                                 echo "##$ip: mdadm -X ${md_disks[@]}" >> $logdir/$logfile
 177                                 ssh $ip "mdadm -X ${md_disks[@]}" >> $logdir/$logfile
 178                                 echo "##$ip: mdadm -E ${md_disks[@]}" >> $logdir/$logfile
 179                                 ssh $ip "mdadm -E ${md_disks[@]}" >> $logdir/$logfile
 180                         fi
 181                 else
 182                         echo "##$ip: no array assembled!" >> $logdir/$logfile
 183                 fi
 184         done
 185         [ "$1" == "fail" ] &&
 186                 echo "See $logdir/$_basename.log and $logdir/$logfile for details"
 187         stop_md all
 188 }
 189
 190 do_setup()
 191 {
 192         check_env
 193         ulimit -c unlimited
 194 }
 195
 196 do_clean()
 197 {
 198         for ip in $NODE1 $NODE2
 199         do
 200                 ssh $ip "mdadm -Ssq; dmesg -c > /dev/null"
 201         done
 202         mdadm --zero ${devlist[@]} &> /dev/null
 203 }
 204
 205 cleanup()
 206 {
 207         check_ssh
 208         do_clean
 209 }
 210
 211 # check: $1/cluster_node $2/feature $3/optional
 212 check()
 213 {
 214         NODES=()
 215         if [ "$1" == "all" ]
 216         then
 217                 NODES=($NODE1 $NODE2)
 218         elif [ "$1" == "$NODE1" -o "$1" == "$NODE2" ]
 219         then
 220                 NODES=$1
 221         else
 222                 die "$1: unknown parameter."
 223         fi
 224         case $2 in
 225                 spares )
 226                         for ip in ${NODES[@]}
 227                         do
 228                                 spares=$(ssh $ip "tr '] ' '\012\012' < /proc/mdstat | grep -c '(S)'")
 229                                 [ "$spares" -ne "$3" ] &&
 230                                         die "$ip: expected $3 spares, but found $spares"
 231                         done
 232                 ;;
 233                 raid* )
 234                         for ip in ${NODES[@]}
 235                         do
 236                                 ssh $ip "grep -sq "$2" /proc/mdstat" ||
 237                                         die "$ip: check '$2' failed."
 238                         done
 239                 ;;
 240                 PENDING | recovery | resync | reshape )
 241                         cnt=5
 242                         for ip in ${NODES[@]}
 243                         do
 244                                 while ! ssh $ip "grep -sq '$2' /proc/mdstat"
 245                                 do
 246                                         if [ "$cnt" -gt '0' ]
 247                                         then
 248                                                 sleep 0.2
 249                                                 cnt=$[cnt-1]
 250                                         else
 251                                                 die "$ip: no '$2' happening!"
 252                                         fi
 253                                 done
 254                         done
 255                 ;;
 256                 wait )
 257                         local cnt=60
 258                         for ip in ${NODES[@]}
 259                         do
 260                                 p=$(ssh $ip "cat /proc/sys/dev/raid/speed_limit_max")
 261                                 ssh $ip "echo 200000 > /proc/sys/dev/raid/speed_limit_max"
 262                                 while ssh $ip "grep -Esq '(resync|recovery|reshape|check|repair)' /proc/mdstat"
 263                                 do
 264                                         if [ "$cnt" -gt '0' ]
 265                                         then
 266                                                 sleep 5
 267                                                 cnt=$[cnt-1]
 268                                         else
 269                                                 die "$ip: Check '$2' timeout over 300 seconds."
 270                                         fi
 271                                 done
 272                                 ssh $ip "echo $p > /proc/sys/dev/raid/speed_limit_max"
 273                         done
 274                 ;;
 275                 bitmap )
 276                         for ip in ${NODES[@]}
 277                         do
 278                                 ssh $ip "grep -sq '$2' /proc/mdstat" ||
 279                                         die "$ip: no '$2' found in /proc/mdstat."
 280                         done
 281                 ;;
 282                 nobitmap )
 283                         for ip in ${NODES[@]}
 284                         do
 285                                 ssh $ip "grep -sq 'bitmap' /proc/mdstat" &&
 286                                         die "$ip: 'bitmap' found in /proc/mdstat."
 287                         done
 288                 ;;
 289                 chunk )
 290                         for ip in ${NODES[@]}
 291                         do
 292                                 chunk_size=`awk -F',' '/chunk/{print $2}' /proc/mdstat | awk -F'[a-z]' '{print $1}'`
 293                                 [ "$chunk_size" -ne "$3" ] &&
 294                                         die "$ip: chunksize should be $3, but it's $chunk_size"
 295                         done
 296                 ;;
 297                 state )
 298                         for ip in ${NODES[@]}
 299                         do
 300                                 ssh $ip "grep -Esq 'blocks.*\[$3\]\$' /proc/mdstat" ||
 301                                         die "$ip: no '$3' found in /proc/mdstat."
 302                         done
 303                 ;;
 304                 nosync )
 305                         for ip in ${NODES[@]}
 306                         do
 307                                 ssh $ip "grep -Eq '(resync|recovery)' /proc/mdstat" &&
 308                                         die "$ip: resync or recovery is happening!"
 309                         done
 310                 ;;
 311                 readonly )
 312                         for ip in ${NODES[@]}
 313                         do
 314                                 ssh $ip "grep -sq "read-only" /proc/mdstat" ||
 315                                         die "$ip: check '$2' failed!"
 316                         done
 317                 ;;
 318                 dmesg )
 319                         for ip in ${NODES[@]}
 320                         do
 321                                 ssh $ip "dmesg | grep -iq 'error\|call trace\|segfault'" &&
 322                                         die "$ip: check '$2' prints errors!"
 323                         done
 324                 ;;
 325                 * )
 326                         die "unknown parameter $2"
 327                 ;;
 328         esac
 329 }