testing/do-tests

   1 #!/bin/bash
   2 # Automatically execute the strongSwan test cases
   3 #
   4 # Copyright (C) 2004  Eric Marchionni, Patrik Rayo
   5 # Zuercher Hochschule Winterthur
   6 #
   7 # This program is free software; you can redistribute it and/or modify it
   8 # under the terms of the GNU General Public License as published by the
   9 # Free Software Foundation; either version 2 of the License, or (at your
  10 # option) any later version.  See <http://www.fsf.org/copyleft/gpl.txt>.
  11 #
  12 # This program is distributed in the hope that it will be useful, but
  13 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  14 # or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 # for more details.
  16
  17 DIR=$(dirname `readlink -f $0`)
  18 . $DIR/testing.conf
  19 . $DIR/scripts/function.sh
  20 SSHCONF="-F $DIR/ssh_config"
  21
  22 [ -d $DIR/hosts ] || die "Directory 'hosts' not found"
  23 [ -d $DIR/tests ] || die "Directory 'tests' not found"
  24 [ -d $BUILDDIR ] ||
  25         die "Directory '$BUILDDIR' does not exist, please run make-testing first"
  26 running_any $STRONGSWANHOSTS || die "Please start test environment before running $0"
  27
  28 ln -sfT $DIR $TESTDIR/testing
  29
  30 ##############################################################################
  31 # take care of new path and file variables
  32 #
  33
  34 [ -d $TESTRESULTSDIR ] || mkdir $TESTRESULTSDIR
  35
  36 TESTDATE=`date +%Y%m%d-%H%M-%S`
  37
  38 TODAYDIR=$TESTRESULTSDIR/$TESTDATE
  39 mkdir $TODAYDIR
  40 TESTRESULTSHTML=$TODAYDIR/all.html
  41 INDEX=$TODAYDIR/index.html
  42 DEFAULTTESTSDIR=$TESTDIR/testing/tests
  43
  44 SOURCEIP_ROUTING_TABLE=220
  45
  46 testnumber="0"
  47 failed_cnt="0"
  48 passed_cnt="0"
  49 subdir_cnt="0"
  50
  51 ##############################################################################
  52 # parse optional arguments
  53 #
  54 while getopts "v" opt
  55 do
  56         case "$opt" in
  57         v)
  58                 verbose=YES
  59                 ;;
  60         esac
  61 done
  62 shift $((OPTIND-1))
  63
  64
  65 function print_time()
  66 {
  67         [ "$verbose" == "YES" ] && echo "$(date +%T.%N) ~ "
  68 }
  69
  70 ##############################################################################
  71 # copy default tests to $BUILDDIR
  72 #
  73
  74 TESTSDIR=$BUILDDIR/tests
  75 [ -d $TESTSDIR ] || mkdir $TESTSDIR
  76
  77 ##############################################################################
  78 # assign IP for each host to hostname
  79 #
  80
  81 for host in $STRONGSWANHOSTS
  82 do
  83     eval ipv4_${host}="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`"
  84     eval ipv6_${host}="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $1 }' | awk '{ print $1 }'`"
  85
  86     case $host in
  87     moon)
  88         eval ipv4_moon1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  89         eval ipv6_moon1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  90         ;;
  91     sun)
  92         eval ipv4_sun1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  93         eval ipv6_sun1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  94         ;;
  95     alice)
  96         eval ipv4_alice1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  97         eval ipv6_alice1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
  98         ;;
  99     venus)
 100         ;;
 101     bob)
 102         ;;
 103     carol)
 104         eval ipv4_carol1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 105         eval ipv6_carol1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 106          ;;
 107     dave)
 108         eval ipv4_dave1="`echo $HOSTNAMEIPV4 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 109         eval ipv6_dave1="`echo $HOSTNAMEIPV6 | sed -n -e "s/^.*${host},//gp" | awk -F, '{ print $2 }' | awk '{ print $1 }'`"
 110         ;;
 111     winnetou)
 112         ;;
 113     esac
 114 done
 115
 116
 117 ##############################################################################
 118 # open ssh sessions
 119 #
 120 for host in $STRONGSWANHOSTS
 121 do
 122     ssh $SSHCONF -N root@`eval echo \\\$ipv4_$host` >/dev/null 2>&1 &
 123     eval ssh_pid_$host="`echo $!`"
 124     do_on_exit kill `eval echo \\\$ssh_pid_$host`
 125 done
 126
 127 ##############################################################################
 128 # determine actual software versions
 129 #
 130
 131 [ -f $SHAREDDIR/.strongswan-version ] && SWANVERSION=`cat $SHAREDDIR/.strongswan-version`
 132 KERNELVERSION=`ssh $SSHCONF root@\$ipv4_winnetou uname -r 2>/dev/null`
 133
 134 # check if tcpdump supports --immediate-mode
 135 ssh $SSHCONF root@$ipv4_winnetou tcpdump --immediate-mode -c 1 >/dev/null 2>&1
 136 if [ $? -eq 0 ]
 137 then
 138         TCPDUMP_IM=--immediate-mode
 139 fi
 140
 141 ##############################################################################
 142 # create header for the results html file
 143 #
 144
 145 ENVIRONMENT_HEADER=$(cat <<@EOF
 146   <table border="0" cellspacing="2" cellpadding="2">
 147     <tr valign="top">
 148       <td><b>Host</b></td>
 149       <td colspan="3">`uname -a`</td>
 150     </tr>
 151     <tr valign="top">
 152       <td><b>Guest kernel</b></td>
 153       <td colspan="3">$KERNELVERSION</td>
 154     </tr>
 155     <tr valign="top">
 156       <td><b>strongSwan</b></td>
 157       <td colspan="3">$SWANVERSION</td>
 158     </tr>
 159     <tr valign="top">
 160       <td><b>Date</b></td>
 161       <td colspan="3">$TESTDATE</td>
 162     </tr>
 163     <tr>
 164       <td width="100">&nbsp;</td>
 165       <td width="300">&nbsp;</td>
 166       <td width=" 80">&nbsp;</td>
 167       <td >&nbsp;</td>
 168     </tr>
 169 @EOF
 170 )
 171
 172 cat > $INDEX <<@EOF
 173 <html>
 174 <head>
 175   <title>strongSwan KVM Tests</title>
 176 </head>
 177 <body>
 178   <h2>strongSwan KVM Tests</h2>
 179   $ENVIRONMENT_HEADER
 180 @EOF
 181
 182 cat > $TESTRESULTSHTML <<@EOF
 183 <html>
 184 <head>
 185   <title>strongSwan KVM Tests - All Tests</title>
 186 </head>
 187 <body>
 188   <div><a href="index.html">strongSwan KVM Tests</a> / All Tests</div>
 189   <h2>All Tests</h2>
 190   $ENVIRONMENT_HEADER
 191     <tr align="left">
 192       <th>Number</th>
 193       <th>Test</th>
 194       <th align="right">Time [s]</th>
 195       <th>Result</th>
 196     </tr>
 197 @EOF
 198
 199 echo "Guest kernel : $KERNELVERSION"
 200 echo "strongSwan   : $SWANVERSION"
 201 echo "Date         : $TESTDATE"
 202 echo
 203
 204 ##############################################################################
 205 # trap CTRL-C to properly terminate a long run
 206 #
 207
 208 function abort_tests()
 209 {
 210         echo -n "...aborting..." > /dev/tty
 211         aborted=YES
 212 }
 213 trap abort_tests INT
 214
 215 ##############################################################################
 216 # enter specific test directory
 217 #
 218 if [ $# -gt 0 ]
 219 then
 220     TESTS=$(printf "%s\n" $* | sort -u)
 221 else
 222     TESTS=$(ls $DEFAULTTESTSDIR)
 223 fi
 224
 225 for SUBDIR in $TESTS
 226 do
 227     SUBTESTS="`basename $SUBDIR`"
 228
 229         if [ $SUBTESTS = $SUBDIR ]
 230         then
 231                 SUBTESTS="`ls $DEFAULTTESTSDIR/$SUBDIR`"
 232         else
 233                 if [[ $SUBTESTS == *'*'* ]]
 234                 then
 235                         SUBTESTS="`basename -a $DEFAULTTESTSDIR/$SUBDIR`"
 236                 fi
 237                 SUBDIR="`dirname $SUBDIR`"
 238         fi
 239
 240     if [ ! -d $TODAYDIR/$SUBDIR ]
 241     then
 242         mkdir $TODAYDIR/$SUBDIR
 243         if [ $testnumber == 0 ]
 244         then
 245             FIRST="<b>Category</b>"
 246         else
 247             FIRST="&nbsp;"
 248         fi
 249
 250         if [ $subdir_cnt != 0 ]
 251         then
 252             echo "      <td align=\"right\">$subdir_cnt</td>" >> $INDEX
 253             echo "      <td>&nbsp;</td>" >> $INDEX
 254             echo "    </tr>" >> $INDEX
 255             subdir_cnt="0"
 256         fi
 257         echo "    <tr>" >> $INDEX
 258     echo "      <td>$FIRST</td>">> $INDEX
 259     echo "      <td><a href=\"$SUBDIR/index.html\">$SUBDIR</a></td>" >> $INDEX
 260
 261         SUBTESTSINDEX=$TODAYDIR/$SUBDIR/index.html
 262         cat > $SUBTESTSINDEX <<@EOF
 263 <html>
 264 <head>
 265   <title>strongSwan $SUBDIR Tests</title>
 266 </head>
 267 <body>
 268   <div><a href="../index.html">strongSwan KVM Tests</a> / $SUBDIR</div>
 269   <h2>strongSwan $SUBDIR Tests</h2>
 270   <table border="0" cellspacing="2" cellpadding="2">
 271     <tr valign="top">
 272       <td><b>Guest kernel</b></td>
 273       <td colspan="3">$KERNELVERSION</td>
 274     </tr>
 275     <tr valign="top">
 276       <td><b>strongSwan</b></td>
 277       <td colspan="3">$SWANVERSION</td>
 278     </tr>
 279     <tr valign="top">
 280       <td><b>Date</b></td>
 281       <td colspan="3">$TESTDATE</td>
 282     </tr>
 283     <tr>
 284       <td width="100">&nbsp;</td>
 285       <td width="300">&nbsp;</td>
 286       <td width=" 50">&nbsp;</td>
 287       <td >&nbsp;</td>
 288     </tr>
 289     <tr align="left">
 290        <th>Number</th>
 291        <th>Test</th>
 292        <th colspan="2">Result</th>
 293     </tr>
 294 @EOF
 295     fi
 296
 297     for name in $SUBTESTS
 298     do
 299         let "testnumber += 1"
 300         let "subdir_cnt += 1"
 301         testname=$SUBDIR/$name
 302         log_action " $testnumber $testname:"
 303
 304         teststart=$(date +%s)
 305
 306         if [ ! -d $DEFAULTTESTSDIR/${testname} ]
 307         then
 308             echo "is missing..skipped"
 309             continue
 310         fi
 311
 312         [ -f $DEFAULTTESTSDIR/${testname}/description.txt ] || die "!! File 'description.txt' is missing"
 313         [ -f $DEFAULTTESTSDIR/${testname}/test.conf ]       || die "!! File 'test.conf' is missing"
 314         [ -f $DEFAULTTESTSDIR/${testname}/pretest.dat ]     || die "!! File 'pretest.dat' is missing"
 315         [ -f $DEFAULTTESTSDIR/${testname}/posttest.dat ]    || die "!! File 'posttest.dat' is missing"
 316         [ -f $DEFAULTTESTSDIR/${testname}/evaltest.dat ]    || die "!! File 'evaltest.dat' is missing"
 317
 318         TESTRESULTDIR=$TODAYDIR/$testname
 319         mkdir -p $TESTRESULTDIR
 320         CONSOLE_LOG=$TESTRESULTDIR/console.log
 321         touch $CONSOLE_LOG
 322
 323         TESTDIR=$TESTSDIR/${testname}
 324
 325         ##########################################################################
 326         # copy test specific configurations to hosts and clear log files
 327         #
 328
 329         DBDIR=/etc/db.d
 330
 331         $DIR/scripts/load-testconfig $testname
 332         unset RADIUSHOSTS
 333         unset DBHOSTS
 334         unset IPV6
 335         unset SWANCTL
 336         source $TESTDIR/test.conf
 337
 338
 339         ##########################################################################
 340         # run tcpdump in the background
 341         #
 342
 343         if [ "$TCPDUMPHOSTS" != "" ]
 344         then
 345             echo -e "TCPDUMP\n" >> $CONSOLE_LOG 2>&1
 346
 347             for host_iface in $TCPDUMPHOSTS
 348             do
 349                 host=`echo $host_iface | awk -F ":" '{print $1}'`
 350                 iface=`echo $host_iface | awk -F ":" '{if ($2 != "") { print $2 } else { printf("eth0") }}'`
 351                 tcpdump_cmd="tcpdump -l $TCPDUMP_IM -i $iface not port ssh and not port domain >/tmp/tcpdump.log 2>/tmp/tcpdump.err.log &"
 352                 echo "$(print_time)${host}# $tcpdump_cmd" >> $CONSOLE_LOG
 353                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host '$tcpdump_cmd'`
 354                 eval TDUP_${host}="true"
 355             done
 356         fi
 357
 358         ##########################################################################
 359         # create database directory in RAM
 360         #
 361
 362         for host in $DBHOSTS
 363         do
 364                 eval HOSTLOGIN=root@\$ipv4_${host}
 365             ssh $SSHCONF $HOSTLOGIN "mkdir -p $DBDIR; mount -t ramfs -o size=5m ramfs $DBDIR" >/dev/null 2>&1
 366             ssh $SSHCONF $HOSTLOGIN "chgrp www-data $DBDIR; chmod g+w $DBDIR" >/dev/null 2>&1
 367         done
 368
 369         ##########################################################################
 370         # flush conntrack table on all hosts
 371         #
 372
 373         for host in $STRONGSWANHOSTS
 374         do
 375                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'conntrack -F' >/dev/null 2>&1
 376         done
 377
 378         ##########################################################################
 379         # remove leak detective log on all hosts
 380         #
 381
 382         export LEAK_DETECTIVE_LOG=/var/log/leak-detective.log
 383         for host in $STRONGSWANHOSTS
 384         do
 385                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'rm -f $LEAK_DETECTIVE_LOG' >/dev/null 2>&1
 386         done
 387
 388         ##########################################################################
 389         # flush IPsec state on all hosts
 390         #
 391
 392         for host in $STRONGSWANHOSTS
 393         do
 394                 ssh $SSHCONF root@`eval echo \\\$ipv4_$host` 'ip xfrm state flush; ip xfrm policy flush' >/dev/null 2>&1
 395         done
 396
 397         ##########################################################################
 398         # execute pre-test commands
 399         #
 400
 401         echo -n "pre.."
 402         echo -e "\nPRE-TEST\n" >> $CONSOLE_LOG 2>&1
 403
 404         eval `awk -F "::" '{
 405             if ($1 !~ /^#.*/ && $2 != "")
 406             {
 407                 printf("echo \"$(print_time)%s# %s\"; ", $1, $2)
 408                 printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2)
 409                 printf("echo;\n")
 410             }
 411         }' $TESTDIR/pretest.dat` >> $CONSOLE_LOG 2>&1
 412
 413
 414         ##########################################################################
 415         # stop tcpdump
 416         #
 417
 418         function stop_tcpdump {
 419             # wait for packets to get processed, but don't wait longer than 1s
 420             eval ssh $SSHCONF root@\$ipv4_${1} "\"i=100; while [ \\\$i -gt 0 ]; do pkill -USR1 tcpdump; tail -1 /tmp/tcpdump.err.log | perl -n -e '/(\\d+).*?(\\d+)/; exit (\\\$1 == \\\$2)' || break; sleep 0.01; i=\\\$((\\\$i-1)); done;\""
 421             echo "$(print_time)${1}# killall tcpdump" >> $CONSOLE_LOG
 422             eval ssh $SSHCONF root@\$ipv4_${1} "\"killall tcpdump; while true; do killall -q -0 tcpdump || break; sleep 0.01; done;\""
 423             eval TDUP_${1}="false"
 424             echo "" >> $CONSOLE_LOG
 425         }
 426
 427
 428         ##########################################################################
 429         # get and evaluate test results
 430         #
 431
 432         echo -n "test.."
 433         echo -e "\nTEST\n" >> $CONSOLE_LOG 2>&1
 434
 435         STATUS="passed"
 436
 437         eval `awk -F "::" '{
 438                 host=$1
 439                 command=$2
 440                 pattern=$3
 441                 hit=$4
 442                 if (host ~ /^#.*/ || command == "")
 443                 {
 444                         next
 445                 }
 446                 printf("cmd_err=\044(tempfile -p test -s err); ")
 447                 printf("cmd_out=\044(tempfile -p test -s out); ")
 448                 if (command == "tcpdump")
 449                 {
 450                         printf("if [ \044TDUP_%s == \"true\" ]; then stop_tcpdump %s; fi; \n", host, host)
 451                         printf("ssh \044SSHCONF root@\044ipv4_%s cat /tmp/tcpdump.log > \044cmd_out; ", host)
 452                 }
 453                 else
 454                 {
 455                         printf("ssh \044SSHCONF root@\044ipv4_%s %s >\044cmd_out 2>\044cmd_err; ",  host, command)
 456                 }
 457                 printf("cmd_res=\044(cat \044cmd_out | grep \"%s\"); ", pattern)
 458                 printf("cmd_exit=\044?; ")
 459                 printf("cmd_fail=0; ")
 460                 if (hit ~ /^[0-9]+$/)
 461                 {
 462                         printf("if [ \044(echo \"\044cmd_res\" | wc -l) -ne %d ] ", hit)
 463                 }
 464                 else
 465                 {
 466                         printf("if [ \044cmd_exit -eq 0 -a \"%s\" = \"NO\"  ] ", hit)
 467                         printf("|| [ \044cmd_exit -ne 0 -a \"%s\" = \"YES\" ] ", hit)
 468                 }
 469                 printf("; then STATUS=\"failed\"; cmd_fail=1; fi; \n")
 470
 471                 printf("if [ \044cmd_fail -ne 0 ]; then echo \"~~~~~~~ FAIL ~~~~~~~\"; fi; \n")
 472                 if (command == "tcpdump")
 473                 {
 474                         printf("echo \"$(print_time)%s# cat /tmp/tcpdump.log | grep \047%s\047  [%s]\"; ", host, pattern, hit)
 475                 }
 476                 else
 477                 {
 478                         printf("echo \"$(print_time)%s# %s | grep \047%s\047  [%s]\"; ", host, command, pattern, hit)
 479                 }
 480                 printf("if [ -n \"\044cmd_res\" ]; then echo \"\044cmd_res\"; fi; \n")
 481                 printf("cat \044cmd_err; \n")
 482                 printf("if [ \044cmd_fail -ne 0 ]; then \n")
 483                 printf("if [ -s \044cmd_out ]; then echo \"~~ output ~~~~~~~~~~\"; \n")
 484                 printf("if [ \"\044verbose\" == \"YES\" ]; then cat \044cmd_out;\n")
 485                 printf("else cat \044cmd_out | head; fi; fi; \n")
 486                 printf("echo \"~~~~~~~~~~~~~~~~~~~~\"; fi; \n")
 487                 printf("rm -f -- \044cmd_out \044cmd_err; \n")
 488                 printf("echo; ")
 489         }' $TESTDIR/evaltest.dat` >> $CONSOLE_LOG 2>&1
 490
 491
 492         ##########################################################################
 493         # log statusall and listall output
 494         # get copies of ipsec.conf, ipsec.secrets
 495         # create index.html for the given test case
 496
 497         cat > $TESTRESULTDIR/index.html <<@EOF
 498 <html>
 499 <head>
 500   <title>Test $testname</title>
 501 </head>
 502 <body>
 503 <table border="0" cellpadding="0" cellspacing="0" width="600">
 504   <tr><td>
 505     <div><a href="../../index.html">strongSwan KVM Tests</a> / <a href="../index.html">$SUBDIR</a> / $name</div>
 506     <h2>Test $testname</h2>
 507     <h3>Description</h3>
 508 @EOF
 509
 510         cat $TESTDIR/description.txt >> $TESTRESULTDIR/index.html
 511
 512         cat >> $TESTRESULTDIR/index.html <<@EOF
 513     <ul>
 514       <li><a href="console.log">console.log</a></li>
 515     </ul>
 516     <img src="../../images/$DIAGRAM" alt="$VIRTHOSTS">
 517 @EOF
 518
 519         IPTABLES_CMD_V4="echo -e '=== filter table ==='; iptables  -v -n -L; echo -e '\n=== nat table ==='; iptables  -v -n -t nat -L; echo -e '\n=== mangle table ==='; iptables  -v -n -t mangle -L"
 520         IPTABLES_CMD_V6="echo -e '=== filter table ==='; ip6tables -v -n -L; echo -e '\n=== nat table ==='; ip6tables -v -n -t nat -L; echo -e '\n=== mangle table ==='; ip6tables -v -n -t mangle -L"
 521
 522         if [ -n "$IPV6" ]
 523         then
 524             IPROUTE_CMD="ip -6 route list table $SOURCEIP_ROUTING_TABLE"
 525             IPROUTE_DSP=$IPROUTE_CMD
 526             IPTABLES_CMD="$IPTABLES_CMD_V6"
 527             IPTABLES_DSP="ip6tables -L"
 528             IPTABLES_SAVE_CMD="ip6tables-save"
 529             IPTABLES_SAVE_DSP="ip6tables-save"
 530         else
 531             IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE"
 532             IPROUTE_DSP=$IPROUTE_CMD
 533             IPTABLES_CMD="$IPTABLES_CMD_V4"
 534             IPTABLES_DSP="iptables -L"
 535             IPTABLES_SAVE_CMD="iptables-save"
 536             IPTABLES_SAVE_DSP="iptables-save"
 537         fi
 538
 539         if [ $name = "net2net-ip4-in-ip6-ikev2" -o $name = "net2net-ip6-in-ip4-ikev2" ]
 540         then
 541             IPROUTE_CMD="ip route list table $SOURCEIP_ROUTING_TABLE; echo; ip -6 route list table $SOURCEIP_ROUTING_TABLE"
 542             IPROUTE_DSP="ip (-6) route list table $SOURCEIP_ROUTING_TABLE"
 543             IPTABLES_CMD="$IPTABLES_CMD_V4; echo; $IPTABLES_CMD_V6"
 544             IPTABLES_DSP="iptables -L ; ip6tables -L"
 545             IPTABLES_SAVE_CMD="iptables-save; echo; ip6tables-save"
 546             IPTABLES_SAVE_DSP="iptables-save ; ip6tables-save"
 547         fi
 548
 549         for host in $DBHOSTS
 550         do
 551             eval HOSTLOGIN=root@\$ipv4_${host}
 552
 553                 scp $SSHCONF $HOSTLOGIN:/etc/db.d/ipsec.sql \
 554                     $TESTRESULTDIR/${host}.ipsec.sql  > /dev/null 2>&1
 555         done
 556
 557         for host in $IPSECHOSTS
 558         do
 559             eval HOSTLOGIN=root@\$ipv4_${host}
 560
 561                 scp $SSHCONF $HOSTLOGIN:/etc/strongswan.conf \
 562                     $TESTRESULTDIR/${host}.strongswan.conf  > /dev/null 2>&1
 563                 if [  -n "$SWANCTL" ]
 564                 then
 565                         scp $SSHCONF $HOSTLOGIN:/etc/swanctl/swanctl.conf \
 566                         $TESTRESULTDIR/${host}.swanctl.conf  > /dev/null 2>&1
 567
 568                         for subsys in conns algs certs pools authorities sas pols
 569                         do
 570                                 ssh $SSHCONF $HOSTLOGIN swanctl --list-$subsys \
 571                                         > $TESTRESULTDIR/${host}.swanctl.$subsys 2>/dev/null
 572                         done
 573
 574                         ssh $SSHCONF $HOSTLOGIN swanctl --stats \
 575                                 > $TESTRESULTDIR/${host}.swanctl.stats 2>/dev/null
 576
 577                         echo "" >> $TESTRESULTDIR/${host}.swanctl.sas
 578                         cat $TESTRESULTDIR/${host}.swanctl.pols >> \
 579                                 $TESTRESULTDIR/${host}.swanctl.sas
 580                         cat $TESTRESULTDIR/${host}.swanctl.algs >> \
 581                                 $TESTRESULTDIR/${host}.swanctl.stats
 582                 else
 583                     for file in ipsec.conf ipsec.secrets
 584                     do
 585                                 scp $SSHCONF $HOSTLOGIN:/etc/$file \
 586                             $TESTRESULTDIR/${host}.$file  > /dev/null 2>&1
 587                     done
 588
 589                     for command in statusall listall
 590                     do
 591                                 ssh $SSHCONF $HOSTLOGIN ipsec $command \
 592                             > $TESTRESULTDIR/${host}.$command 2>/dev/null
 593                     done
 594                 fi
 595
 596                 if (! [ -f $TESTRESULTDIR/${host}.ipsec.sql ] ) then
 597                         scp $SSHCONF $HOSTLOGIN:/etc/ipsec.d/ipsec.sql \
 598                                 $TESTRESULTDIR/${host}.ipsec.sql  > /dev/null 2>&1
 599                 fi
 600
 601             ssh $SSHCONF $HOSTLOGIN ip -s xfrm policy \
 602                     > $TESTRESULTDIR/${host}.ip.policy 2>/dev/null
 603             ssh $SSHCONF $HOSTLOGIN ip -s xfrm state \
 604                     > $TESTRESULTDIR/${host}.ip.state 2>/dev/null
 605             ssh $SSHCONF $HOSTLOGIN $IPROUTE_CMD \
 606                     > $TESTRESULTDIR/${host}.ip.route 2>/dev/null
 607             ssh $SSHCONF $HOSTLOGIN $IPTABLES_CMD \
 608                     > $TESTRESULTDIR/${host}.iptables 2>/dev/null
 609                 ssh $SSHCONF $HOSTLOGIN $IPTABLES_SAVE_CMD \
 610                     > $TESTRESULTDIR/${host}.iptables-save 2>/dev/null
 611             chmod a+r $TESTRESULTDIR/*
 612
 613                 if [ -n "$SWANCTL" ]
 614                 then
 615                     cat >> $TESTRESULTDIR/index.html <<@EOF
 616     <h3>$host</h3>
 617       <table border="0" cellspacing="0" width="600">
 618       <tr>
 619         <td valign="top">
 620           <ul>
 621             <li><a href="$host.swanctl.conf">swanctl.conf</a></li>
 622             <li><a href="$host.swanctl.conns">swanctl --list-conns</a></li>
 623             <li><a href="$host.swanctl.certs">swanctl --list-certs</a></li>
 624             <li><a href="$host.strongswan.conf">strongswan.conf</a></li>
 625             <li><a href="$host.ipsec.sql">ipsec.sql</a></li>
 626           </ul>
 627         </td>
 628         <td valign="top">
 629           <ul>
 630             <li><a href="$host.swanctl.sas">swanctl --list-sas|--list-pols</a></li>
 631             <li><a href="$host.swanctl.pools">swanctl --list-pools</a></li>
 632             <li><a href="$host.swanctl.authorities">swanctl --list-authorities</a></li>
 633             <li><a href="$host.swanctl.stats">swanctl --stats|--list-algs</a></li>
 634             <li><a href="$host.auth.log">auth.log</a></li>
 635             <li><a href="$host.daemon.log">daemon.log</a></li>
 636           </ul>
 637       </td>
 638         <td valign="top">
 639           <ul>
 640             <li><a href="$host.ip.policy">ip -s xfrm policy</a></li>
 641             <li><a href="$host.ip.state">ip -s xfrm state</a></li>
 642             <li><a href="$host.ip.route">$IPROUTE_DSP</a></li>
 643             <li><a href="$host.iptables">$IPTABLES_DSP</a></li>
 644             <li><a href="$host.iptables-save">$IPTABLES_SAVE_DSP</a></li>
 645           </ul>
 646           &nbsp;
 647       </td>
 648     </tr>
 649     </table>
 650 @EOF
 651
 652                 else
 653                     cat >> $TESTRESULTDIR/index.html <<@EOF
 654     <h3>$host</h3>
 655       <table border="0" cellspacing="0" width="600">
 656       <tr>
 657         <td valign="top">
 658           <ul>
 659             <li><a href="$host.ipsec.conf">ipsec.conf</a></li>
 660             <li><a href="$host.ipsec.secrets">ipsec.secrets</a></li>
 661             <li><a href="$host.ipsec.sql">ipsec.sql</a></li>
 662             <li><a href="$host.strongswan.conf">strongswan.conf</a></li>
 663           </ul>
 664         </td>
 665         <td valign="top">
 666           <ul>
 667             <li><a href="$host.statusall">ipsec statusall</a></li>
 668             <li><a href="$host.listall">ipsec listall</a></li>
 669             <li><a href="$host.auth.log">auth.log</a></li>
 670             <li><a href="$host.daemon.log">daemon.log</a></li>
 671           </ul>
 672       </td>
 673         <td valign="top">
 674           <ul>
 675             <li><a href="$host.ip.policy">ip -s xfrm policy</a></li>
 676             <li><a href="$host.ip.state">ip -s xfrm state</a></li>
 677             <li><a href="$host.ip.route">$IPROUTE_DSP</a></li>
 678             <li><a href="$host.iptables">$IPTABLES_DSP</a></li>
 679             <li><a href="$host.iptables-save">$IPTABLES_SAVE_DSP</a></li>
 680           </ul>
 681       </td>
 682     </tr>
 683     </table>
 684 @EOF
 685                 fi
 686         done
 687
 688         for host in $RADIUSHOSTS
 689         do
 690             eval HOSTLOGIN=root@\$ipv4_${host}
 691
 692             for file in clients.conf eap.conf radiusd.conf proxy.conf users
 693             do
 694                 scp $SSHCONF $HOSTLOGIN:/etc/freeradius/$file \
 695                     $TESTRESULTDIR/${host}.$file  > /dev/null 2>&1
 696             done
 697
 698                 scp $SSHCONF $HOSTLOGIN:/etc/strongswan.conf \
 699                     $TESTRESULTDIR/${host}.strongswan.conf  > /dev/null 2>&1
 700
 701             scp $SSHCONF $HOSTLOGIN:/var/log/freeradius/radius.log \
 702                 $TESTRESULTDIR/${host}.radius.log  > /dev/null 2>&1
 703
 704             ssh $SSHCONF $HOSTLOGIN grep imcv /var/log/daemon.log \
 705                 >> $TESTRESULTDIR/${host}.daemon.log 2>/dev/null
 706
 707             chmod a+r $TESTRESULTDIR/*
 708             cat >> $TESTRESULTDIR/index.html <<@EOF
 709     <h3>$host</h3>
 710       <table border="0" cellspacing="0" width="600">
 711       <tr>
 712         <td valign="top">
 713           <ul>
 714             <li><a href="$host.clients.conf">clients.conf</a></li>
 715             <li><a href="$host.radiusd.conf">radiusd.conf</a></li>
 716             <li><a href="$host.strongswan.conf">strongswan.conf</a></li>
 717           </ul>
 718         </td>
 719         <td valign="top">
 720           <ul>
 721             <li><a href="$host.eap.conf">eap.conf</a></li>
 722             <li><a href="$host.radius.log">radius.log</a></li>
 723             <li><a href="$host.daemon.log">daemon.log</a></li>
 724           </ul>
 725       </td>
 726         <td valign="top">
 727           <ul>
 728             <li><a href="$host.proxy.conf">proxy.conf</a></li>
 729             <li><a href="$host.users">users</a></li>
 730           </ul>
 731       </td>
 732     </tr>
 733     </table>
 734 @EOF
 735
 736         done
 737
 738         cat >> $TESTRESULTDIR/index.html <<@EOF
 739         <h3>tcpdump</h3>
 740         <ul>
 741 @EOF
 742
 743         for host in $TCPDUMPHOSTS
 744         do
 745             cat >> $TESTRESULTDIR/index.html <<@EOF
 746             <li><a href="$host.tcpdump.log">$host tcpdump.log</a></li>
 747 @EOF
 748         done
 749
 750         cat >> $TESTRESULTDIR/index.html <<@EOF
 751         </ul>
 752 @EOF
 753
 754         cat >> $TESTRESULTDIR/index.html <<@EOF
 755   </td></tr>
 756 </table>
 757 </body>
 758 </html>
 759 @EOF
 760
 761
 762         ##########################################################################
 763         # execute post-test commands
 764         #
 765
 766         echo -n "post"
 767         echo -e "\nPOST-TEST\n" >> $CONSOLE_LOG 2>&1
 768
 769         eval `awk -F "::" '{
 770             if ($1 !~ /^#.*/ && $2 != "")
 771             {
 772                 printf("echo \"$(print_time)%s# %s\"; ", $1, $2)
 773                 printf("ssh \044SSHCONF root@\044ipv4_%s \"%s\"; ", $1, $2)
 774                 printf("echo;\n")
 775             }
 776         }' $TESTDIR/posttest.dat` >> $CONSOLE_LOG 2>&1
 777
 778         ##########################################################################
 779         # check that IPsec state was cleaned up properly
 780         #
 781
 782         for host in $IPSECHOSTS
 783         do
 784                 eval HOSTLOGIN=root@\$ipv4_${host}
 785                 IPSECSTATE=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm state'`
 786                 # ignore IPv4/v6 states created with IPComp SAs
 787                 IPSECSTATEISSUE=`echo "$IPSECSTATE" | grep 'proto.*spi' | grep -v 'proto 4'`
 788                 IPSECPOLICY=`ssh $SSHCONF $HOSTLOGIN 'ip xfrm policy'`
 789                 if [ -n "$IPSECSTATEISSUE" -o -n "$IPSECPOLICY" ]
 790                 then
 791                         echo -e "\n$host# ip xfrm state [NO]" >> $CONSOLE_LOG
 792                         echo "$IPSECSTATE" >> $CONSOLE_LOG
 793                         echo -e "\n$host# ip xfrm policy [NO]" >> $CONSOLE_LOG
 794                         echo "$IPSECPOLICY" >> $CONSOLE_LOG
 795                         STATUS="failed"
 796                 fi
 797         done
 798
 799
 800         ##########################################################################
 801         # make sure there were no leaks
 802         #
 803
 804         for host in $STRONGSWANHOSTS
 805         do
 806                 eval HOSTLOGIN=root@\$ipv4_${host}
 807                 LEAKS=`ssh $SSHCONF $HOSTLOGIN 'cat $LEAK_DETECTIVE_LOG 2>/dev/null | grep -v "No leaks detected.*"'`
 808                 if [ -n "$LEAKS" ]
 809                 then
 810                         echo -e "\n$host# cat $LEAK_DETECTIVE_LOG [NO]" >> $CONSOLE_LOG
 811                         echo "$LEAKS" >> $CONSOLE_LOG
 812                         echo "<<< $host $LEAK_DETECTIVE_LOG >>>" >> $CONSOLE_LOG
 813                         STATUS="failed"
 814                 fi
 815         done
 816
 817
 818         ##########################################################################
 819         # get a copy of /var/log/auth.log
 820         #
 821
 822         for host in $IPSECHOSTS
 823         do
 824             eval HOSTLOGIN=root@\$ipv4_${host}
 825             ssh $SSHCONF $HOSTLOGIN "grep -s -E 'charon|last message repeated|imcv|pt-tls-client' \
 826                 /var/log/auth.log" >> $TESTRESULTDIR/${host}.auth.log
 827         done
 828
 829
 830         ##########################################################################
 831         # get a copy of /var/log/daemon.log
 832         #
 833
 834         for host in $IPSECHOSTS
 835         do
 836             eval HOSTLOGIN=root@\$ipv4_${host}
 837             ssh $SSHCONF $HOSTLOGIN "grep -s -E 'systemd|swanctl|charon|last message repeated|imcv' \
 838                 /var/log/daemon.log" >> $TESTRESULTDIR/${host}.daemon.log
 839         done
 840
 841
 842         ##########################################################################
 843         # stop tcpdump if necessary
 844         #
 845
 846         for host in $TCPDUMPHOSTS
 847         do
 848             if [ "`eval echo \\\$TDUP_${host}`" = "true" ]
 849             then
 850                         stop_tcpdump $host
 851             fi
 852             eval HOSTLOGIN=root@\$ipv4_${host}
 853                 scp $SSHCONF $HOSTLOGIN:/tmp/tcpdump.log \
 854                         $TESTRESULTDIR/${host}.tcpdump.log > /dev/null 2>&1
 855         done
 856
 857         ##########################################################################
 858         # remove database directory if needed
 859         #
 860
 861         for host in $DBHOSTS
 862         do
 863                 eval HOSTLOGIN=root@\$ipv4_${host}
 864             ssh $SSHCONF $HOSTLOGIN "umount $DBDIR; rm -r $DBDIR" > /dev/null 2>&1
 865         done
 866
 867         ##########################################################################
 868         # copy default host config back if necessary
 869         #
 870
 871         $DIR/scripts/restore-defaults $testname
 872
 873
 874         ##########################################################################
 875         # set counters
 876         #
 877
 878         if [ $STATUS = "failed" ]
 879         then
 880             let "failed_cnt += 1"
 881         else
 882             let "passed_cnt += 1"
 883         fi
 884
 885
 886         ##########################################################################
 887         # write test status to html file
 888         #
 889         testend=$(date +%s)
 890         let "testend -= teststart"
 891         let "timetotal += testend"
 892
 893         if [ $STATUS = "passed" ]
 894         then
 895                 COLOR="green"
 896                 log_status 0
 897         else
 898                 COLOR="red"
 899                 log_status 1
 900         fi
 901
 902         cat >> $TESTRESULTSHTML << @EOF
 903   <tr>
 904     <td>$testnumber</td>
 905     <td><a href="$testname/index.html">$testname</a></td>
 906     <td align="right">$testend</td>
 907     <td><a href="$testname/console.log"><font color="$COLOR">$STATUS</font></a></td>
 908   </tr>
 909 @EOF
 910         cat >> $SUBTESTSINDEX << @EOF
 911   <tr>
 912     <td>$testnumber</td>
 913     <td><a href="$name/index.html">$name</a></td>
 914     <td><a href="$name/console.log"><font color="$COLOR">$STATUS</font></a></td>
 915     <td>&nbsp;</td>
 916   </tr>
 917 @EOF
 918
 919
 920         ##########################################################################
 921         # remove any charon.pid files that still may exist
 922         #
 923
 924         for host in $IPSECHOSTS
 925         do
 926             eval HOSTLOGIN=root@\$ipv4_${host}
 927             ssh $SSHCONF $HOSTLOGIN 'if [ -f /var/run/charon.pid ]; then rm /var/run/charon.pid; echo "    removed charon.pid on `hostname`"; fi'
 928         done
 929
 930         if [ -n "$aborted" ]
 931         then
 932             break 2
 933         fi
 934
 935     done
 936
 937 done
 938
 939
 940 ##############################################################################
 941 # finish the results html file
 942 #
 943
 944 cat >> $TESTRESULTSHTML << @EOF
 945     <tr>
 946       <td>&nbsp;</td><td>&nbsp;</td><td>&nbsp;</td><td>&nbsp;</td>
 947     </tr>
 948     <tr>
 949       <td><b>Passed</b></td><td><b><font color="green">$passed_cnt</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 950     </tr>
 951     <tr>
 952       <td><b>Failed</b></td><td><b><font color="red">$failed_cnt</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 953     </tr>
 954     <tr>
 955       <td><b>Time [s]</b></td><td><b><font color="blue">$timetotal</font></b></td><td>&nbsp;</td><td>&nbsp;</td>
 956     </tr>
 957   </table>
 958 </body>
 959 </html>
 960 @EOF
 961
 962 if [ $subdir_cnt != 0 ]
 963 then
 964 cat >> $INDEX << @EOF
 965       <td align="right">$subdir_cnt</td>
 966       <td>&nbsp;</td>
 967     </tr>
 968 @EOF
 969 fi
 970
 971 let "all_cnt = $passed_cnt + $failed_cnt"
 972
 973 cat >> $INDEX << @EOF
 974     <tr>
 975       <td>&nbsp;</td>
 976       <td><a href="all.html"><b>all</b></a></td>
 977       <td align="right"><b>$all_cnt</b></td>
 978       <td>&nbsp;</td>
 979     </tr>
 980     <tr>
 981       <td><b>Failed</b></td>
 982       <td>&nbsp;</td>
 983       <td align="right"><b><font color="red">$failed_cnt</font></b></td>
 984       <td>&nbsp;</td>
 985     </tr>
 986   </table>
 987 </body>
 988 </html>
 989 @EOF
 990
 991 echo
 992 echo_ok     "Passed : $passed_cnt"
 993 echo_failed "Failed : $failed_cnt"
 994
 995 echo
 996 echo "The results are available in $TODAYDIR"
 997 echo "or via the link http://$ipv4_winnetou/testresults/$TESTDATE"
 998
 999 ENDDATE=`date +%Y%m%d-%H%M-%S`
1000 echo
1001 echo "Finished : $ENDDATE"