Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.

(cherry picked from commit cf5105939c1c1802e69cc63a39e9d71b6e5f67e1)

diff --git a/.gitignore b/.gitignore
index 8c4e3394630eeceb16b69fe03a92a866ea1fce72..c6b8ee52eedf4eba2907f18319a2cc4b5f8f31cf 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 475cec53522f82dd2a3d3f1c82cf4b3913520803..aa717437541aab90d6a014292a064b9149336074 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -195,6 +195,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -252,6 +253,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - ( cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k test V=1 )
     - test -s bin/tests/system/systests.output
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index f51188bc289a988aedf1186b81d45c6d05d6ee89..30b45b094caefa8e73506ef711be0133f78ba958 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*

diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 2317bd8544c7a055025d75b8b4ce566dae40fad0..d6b07a5f048a52f8bf704aef980d858c25429aea 100644 (file)
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -114,6 +114,7 @@ PERL=@PERL@
 PSSUSPEND=
 
 PYTHON=@PYTHON@
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEGEOIP2=${MAXMINDDB_LIBS:+1}
 ZLIB_LIBS="@ZLIB_LIBS@"
 HAVEZLIB=${ZLIB_LIBS:+1}
 NZD=@NZD_TOOLS@
+
+export HAVEXMLSTATS HAVEJSONSTATS

diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 37eb737eefeacd6d4096403baf28117f15f9df0e..a9e2f2652f40e10ec18fd7b03203ff563159e523 100755 (executable)
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -109,6 +109,39 @@ export CONTROLPORT
 export LOWPORT
 export HIGHPORT
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:`date`"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -166,17 +199,40 @@ then
    ( cd $systest && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-$PERL start.pl --port $PORT $systest
-if [ $? -ne 0 ]; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:`date $dateargs`"
-    exit 1
+status=0
+run=0
+# Run the tests
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
+fi
+
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+       start_servers
+       rm -f "$systest/$test.status"
+       test_status=0
+       (cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+       if [ -f "$systest/$test.status" ]; then
+           echo_i "FAILED"
+           test_status=$(cat "$systest/$test.status")
+       fi
+       status=$((status+test_status))
+       stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
 fi
 
-# Run the tests
-( cd $systest ; $SHELL tests.sh "$@" )
-status=$?
 
 if $stopservers
 then

diff --git a/configure.ac b/configure.ac
index 9d0bd6d1124ea2d722047704e0b7c2caf7d865ca..c921c83aac1a0bef5fd8294697f8aaef7beecbba 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -339,6 +339,12 @@ case "$INSTALL" in
                ;;
 esac
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
+
 AC_PROG_CC
 AC_PROG_CC_C99