Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.

diff --git a/.gitignore b/.gitignore
index 71607aa297c9b685c70effb3f1b3e44e985e47b0..742df701d093603a710f54b245b46d3b72e1f380 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0662fc1de3eafceedfbe8e92c819bf939f20874..65a3fc0f8fbd46f36541dc64c7f13d5d37914ee3 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,6 +199,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -256,6 +257,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 0c66e8d2cc9c291b04ea3c13661cef019eaa67ab..d7c9c7b1bc3563c6bf9baa2a952fb2709577efd0 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*

diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 85f8df69e4d1a1e1cdf501d65dda1295743e7959..4088fb4411c88ab18015c3d7484c83ba8a035760 100644 (file)
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -112,6 +112,7 @@ PERL=$(command -v "@PERL@")
 PSSUSPEND=
 
 PYTHON=$(command -v "@PYTHON@")
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEZLIB=${ZLIB_LIBS:+1}
 LMDB_LIBS="@LMDB_LIBS@"
 NZD=${LMDB_LIBS:+1}
 CRYPTO=@CRYPTO@
+
+export HAVEXMLSTATS HAVEJSONSTATS

diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 5e4095ea67d4bac07a794b2f6fdc605a92c1c636..4e8f5cc6b90f526507da832550cbcac387fa4777 100644 (file)
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -108,6 +108,39 @@ fi
 # true, the last digit of EXTRAPORTn is "n".
 eval "$(${srcdir}/get_ports.sh -p "$baseport")"
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:$(date_with_args)"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -161,16 +194,40 @@ then
    ( cd "${systest}" && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-if ! $PERL ${srcdir}/start.pl --port "$PORT" "$systest"; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:$(date_with_args)"
-    exit 1
-fi
 
+status=0
+run=0
 # Run the tests
-( cd "${systest}" && $SHELL tests.sh "$@" )
-status=$?
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
+fi
+
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+       start_servers
+       rm -f "$systest/$test.status"
+       test_status=0
+       (cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+       if [ -f "$systest/$test.status" ]; then
+           echo_i "FAILED"
+           test_status=$(cat "$systest/$test.status")
+       fi
+       status=$((status+test_status))
+       stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
+fi
 
 if $stopservers
 then
@@ -179,11 +236,6 @@ else
     exit $status
 fi
 
-# Shutdown
-$PERL ${srcdir}/stop.pl "$systest"
-
-status=$((status + $?))
-
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.

diff --git a/configure.ac b/configure.ac
index 8ad62afb2709a1193ee7edffa84774f17fb1a579..458bd6cda263b7b57bc7986ad9bb85de598698d1 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -276,6 +276,11 @@ AM_CONDITIONAL([HAVE_PERLMOD_TIME_HIRES],
 AM_PATH_PYTHON([3.4], [], [:])
 AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
 AX_PYTHON_MODULE([dns])
 AM_CONDITIONAL([HAVE_PYMOD_DNS], [test "$HAVE_PYMOD_DNS" = "yes"])