regression-tests/runtests

   1 #!/usr/bin/env bash
   2 if [ -z "$testsdir" ]; then
   3   echo "Incorrect usage. You probably want ./start-test-stop help"
   4   exit 1
   5 fi
   6
   7 PATH=.:$PATH:/usr/sbin
   8 MAKE=${MAKE:-make}
   9
  10 export PDNS=${PDNS:-${PWD}/../pdns/pdns_server}
  11 export PDNS2=${PDNS2:-${PWD}/../pdns/pdns_server}
  12 export PDNSRECURSOR=${PDNSRECURSOR:-${PWD}/../pdns/recursordist/pdns_recursor}
  13 export RECCONTROL=${RECCONTROL:-${PWD}/../pdns/recursordist/rec_control}
  14 export SDIG=${SDIG:-${PWD}/../pdns/sdig}
  15 export NOTIFY=${NOTIFY:-${PWD}/../pdns/pdns_notify}
  16 export NSEC3DIG=${NSEC3DIG:-${PWD}/../pdns/nsec3dig}
  17 export SAXFR=${SAXFR:-${PWD}/../pdns/saxfr}
  18 export ZONE2SQL=${ZONE2SQL:-${PWD}/../pdns/zone2sql}
  19 export ZONE2JSON=${ZONE2JSON:-${PWD}/../pdns/zone2json}
  20 export ZONE2LDAP=${ZONE2LDAP:-${PWD}/../pdns/zone2ldap}
  21 export PDNSUTIL=${PDNSUTIL:-${PWD}/../pdns/pdnsutil}
  22 export PDNSCONTROL=${PDNSCONTROL:-${PWD}/../pdns/pdns_control}
  23
  24 unset _JAVA_OPTIONS
  25
  26 spectest=$1
  27 [ -z $spectest ] && spectest=""
  28
  29 for prog in $SDIG $SAXFR $NOTIFY $NSEC3DIG; do
  30   if `echo $prog | grep -q '\.\./pdns'`; then
  31     ${MAKE} -C ../pdns ${prog##*../pdns/} || exit
  32   fi
  33 done
  34
  35 export SDIG="timeout 5 $SDIG"
  36 export NSEC3DIG="timeout 5 $NSEC3DIG"
  37 export SAXFR="timeout 30 $SAXFR"
  38 export PDNSCONTROL="timeout 5 $PDNSCONTROL"
  39
  40 rm -f test-results failed_tests passed_tests skipped_tests ${testsdir}/*/real_result ${testsdir}/*/diff ${testsdir}/*/*.out ${testsdir}/*/start ${testsdir}/*/step.*
  41
  42 passed=0
  43 failed=0
  44 skipped=0
  45
  46 touch passed_tests failed_tests skipped_tests
  47
  48 for a in $(find $testsdir -type d | grep -v ^.$ | grep -v .svn | grep -v ^confdir | LC_ALL=C sort)
  49 do
  50         if [ ! -x $a/command ]
  51         then
  52                 continue
  53         fi
  54         testname=$(basename $a)
  55         export testname
  56         echo  "$testname: "
  57         if [ "${PDNS_DEBUG}" = "YES" ]; then
  58                 cat $a/description
  59         fi
  60
  61         echo  "$testname: " >> test-results
  62         cat $a/description >> test-results
  63
  64         SKIPIT=0
  65         if [ -e $a/skip ]
  66         then
  67                 SKIPIT=1
  68                 result="        Skipped test $a"
  69         else
  70                 for reason in $skipreasons $context $backend
  71                 do
  72                         if [ -e $a/skip.$reason ]
  73                         then
  74                                 SKIPIT=1
  75                                 result="        Skipped test $a for reason $reason"
  76                                 break
  77                         fi
  78                 done
  79         fi
  80
  81         FAIL=0
  82         for reason in $skipreasons $context $backend
  83         do
  84                 if [ -e $a/fail.$reason ]
  85                 then
  86                         FAIL=1
  87                         break
  88                 fi
  89         done
  90
  91         if [ "$spectest" != "" ] && [ "$spectest" != "$testname" ] && [ "$testname" != "00dnssec-grabkeys" ]
  92         then
  93                 SKIPIT=1
  94                 result="        Skipped test $a because it's not the specified single test"
  95         fi
  96
  97
  98         if [ $SKIPIT = 1 ]
  99         then
 100                 echo $testname >> skipped_tests
 101                 skipped=$[$skipped+1]
 102         else
 103                 $a/command > $a/real_result
 104                 expected=$a/expected_result
 105
 106                 diffopts="-u"
 107                 if [ -e $a/expected_result.i ]; then
 108                         expected=$a/expected_result.i
 109                         diffopts="${diffopts} -i"
 110                 fi
 111
 112                 for extracontext in $extracontexts
 113                 do
 114                         [ -e "$a/expected_result.$extracontext" ] && expected=$a/expected_result.$extracontext
 115                 done
 116                 [ -n "$context" ] && [ -e "$a/expected_result.$context" ] && expected=$a/expected_result.$context
 117                 diff ${diffopts} $expected $a/real_result > $a/diff 2>&1
 118                 if [ -s $a/diff ]
 119                 then
 120                         if [ $FAIL = 0 ]
 121                         then
 122                                 result="        Failed test $a"
 123                                 echo $testname >> failed_tests
 124                                 failed=$[$failed+1]
 125                                 if [ "$FIX_TESTS" == "YES" ]
 126                                 then
 127                                         mv -f $a/real_result $expected
 128                                         result="$result (fixed)"
 129                                 fi
 130                         else
 131                                 result="        Expected failure for reason $reason, test passed $a"
 132                                 echo $testname >> passed_tests
 133                                 passed=$[$passed+1]
 134                         fi
 135                 else
 136                         if [ $FAIL = 0 ]
 137                         then
 138                                 result="        Passed test $a"
 139                                 echo $testname >> passed_tests
 140                                 passed=$[$passed+1]
 141                         else
 142                                 result="        Unexpected pass for reason $reason, test failed $a"
 143                                 echo $testname >> failed_tests
 144                                 failed=$[$failed+1]
 145                         fi
 146                 fi
 147         fi
 148         echo "$result"
 149         echo
 150         echo "$result" >> test-results
 151         echo >> test-results
 152 done
 153
 154 if [ $failed -gt 0 ]; then
 155         echo -n "::error title=Regression-tests::Tests failed. "
 156 fi
 157 echo -n $passed out of $[$passed+$failed]
 158 echo -n " ("
 159 res=$((echo scale=2; echo 100*$passed/\($passed+$failed\)) | bc )
 160 echo -n "$res%) "
 161 echo tests passed, $skipped were skipped