CI: Allow runUnittests to use 'expect' to run the tests

There seems to be a race condition between starting the asterisk
daemon and attempting to use 'asterisk -r' that can cause the
control socket file to not be created.  Since all of the Jenkins
slaves have 'expect' installed, the runUnittests script can use
it to start asterisk in the forground and issue the commands
interactively.  This is much more reliable and it can also make
startup errors more visible since they'll be in the Jenkins console
output.

If 'expect' isn't installed, the original daemon/asterisk -r
process is used.

Also added a "core show settings" before running the tests
and added "notice,warning,error" to the console log.

Change-Id: Idd656085f854afede813ac241b9e312b31358160

diff --git a/tests/CI/runUnittests.sh b/tests/CI/runUnittests.sh
index 682c5064fbfb3ee0b2e6ae9d2174e2dbd120170c..e2d7e45a7ed959adbefeb2b76da62f95087194cd 100755 (executable)
--- a/tests/CI/runUnittests.sh
+++ b/tests/CI/runUnittests.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 CIDIR=$(dirname $(readlink -fn $0))
+NO_EXPECT=0
 source $CIDIR/ci.functions
 ASTETCDIR=$DESTDIR/etc/asterisk
 
@@ -14,9 +15,54 @@ asterisk_corefile_glob() {
        fi
 }
 
+run_tests_expect() {
+$EXPECT <<-EOF
+       spawn sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -fcng -C $CONFFILE
+       match_max 512
+       set timeout 600
+       expect -notransfer "Asterisk Ready."
+       send "core show settings\r"
+       expect -notransfer "CLI>"
+       send "${UNITTEST_COMMAND:-test execute all}\r"
+       expect -notransfer -ex "Test(s) Executed"
+       expect -notransfer "CLI>"
+       send "test show results failed\r"
+       expect -notransfer "CLI>"
+       send "test generate results xml ${OUTPUTFILE}\r"
+       expect -notransfer "CLI>"
+       send "core stop now\r"
+       expect -notransfer "Executing last minute cleanups"
+       wait
+EOF
+}
+
+run_tests_socket() {
+       sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -gn -C $CONFFILE
+       for n in {1..5} ; do
+               sleep 3
+               $ASTERISK -rx "core waitfullybooted" -C $CONFFILE && break
+       done
+       sleep 1
+       $ASTERISK -rx "core show settings" -C $CONFFILE
+       $ASTERISK -rx "${UNITTEST_COMMAND:-test execute all}" -C $CONFFILE
+       $ASTERISK -rx "test show results failed" -C $CONFFILE
+       $ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
+       $ASTERISK -rx "core stop now" -C $CONFFILE
+}
+
+# If DESTDIR is used to install and run asterisk from non standard locations,
+# the directory entries in asterisk.conf need to be munged to prepend DESTDIR.
+ALTERED=$(head -10 ../tmp/DESTDIR/etc/asterisk/asterisk.conf | grep -q "DESTDIR" && echo yes)
+if [ x"$ALTERED" = x ] ; then
+       # In the section that starts with [directories and ends with a blank line,
+       # replace "=> " with "=> ${DESTDIR}"
+       sed -i -r -e "/^\[directories/,/^$/ s@=>\s+@=> ${DESTDIR}@" "$ASTETCDIR/asterisk.conf"
+fi
+
 cat <<-EOF > "$ASTETCDIR/logger.conf"
        [logfiles]
        full => notice,warning,error,debug,verbose
+       console => notice,warning,error
 EOF
 
 echo "[default]" > "$ASTETCDIR/extensions.conf"
@@ -58,6 +104,7 @@ ASTERISK="$DESTDIR/usr/sbin/asterisk"
 CONFFILE=$ASTETCDIR/asterisk.conf
 OUTPUTDIR=${OUTPUT_DIR:-tests/CI/output/}
 OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
+EXPECT="$(which expect 2>/dev/null || : )"
 
 [ ! -d ${OUTPUTDIR} ] && mkdir -p $OUTPUTDIR
 [ x"$USER_GROUP" != x ] && sudo chown -R $USER_GROUP $OUTPUTDIR
@@ -65,16 +112,14 @@ OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
 rm -rf $ASTETCDIR/extensions.{ael,lua} || :
 
 set -x
-sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -gn -C $CONFFILE
-for n in {1..5} ; do
-       sleep 3
-       $ASTERISK -rx "core waitfullybooted" -C $CONFFILE && break
-done
-sleep 1
-$ASTERISK -rx "${UNITTEST_COMMAND:-test execute all}" -C $CONFFILE
-$ASTERISK -rx "test show results failed" -C $CONFFILE
-$ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
-$ASTERISK -rx "core stop now" -C $CONFFILE
+if [ x"$EXPECT" != x -a $NO_EXPECT -eq 0 ] ; then
+       run_tests_expect
+else
+       run_tests_socket
+fi
+
+# Cleanup "just in case"
+sudo killall -qe -ABRT $ASTERISK 
 
 runner rsync -vaH $DESTDIR/var/log/asterisk/. $OUTPUTDIR
 set +x
@@ -86,6 +131,7 @@ do
        if [ -f $core ]
        then
                echo "*** Found a core file ($core) after running unit tests ***"
+               set -x
                sudo OUTPUTDIR=$OUTPUTDIR $DESTDIR/var/lib/asterisk/scripts/ast_coredumper --no-default-search $core
        fi
 done