From: Ondřej Surý <ondrej@sury.org>
Date: Fri, 27 Mar 2020 12:23:24 +0000 (+0100)
Subject: Add a pytest runner to run.sh
X-Git-Tag: v9.16.3~27^2~1
X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=a4f0281962c8d8f80f6efc04cad550146eec8463;p=thirdparty%2Fbind9.git

Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.

(cherry picked from commit cf5105939c1c1802e69cc63a39e9d71b6e5f67e1)
---

diff --git a/.gitignore b/.gitignore
index 8c4e3394630..c6b8ee52eed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 475cec53522..aa717437541 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -195,6 +195,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -252,6 +253,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - ( cd bin/tests/system && make -j${TEST_PARALLEL_JOBS:-1} -k test V=1 )
     - test -s bin/tests/system/systests.output
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')
diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index f51188bc289..30b45b094ca 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 2317bd8544c..d6b07a5f048 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -114,6 +114,7 @@ PERL=@PERL@
 PSSUSPEND=
 
 PYTHON=@PYTHON@
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEGEOIP2=${MAXMINDDB_LIBS:+1}
 ZLIB_LIBS="@ZLIB_LIBS@"
 HAVEZLIB=${ZLIB_LIBS:+1}
 NZD=@NZD_TOOLS@
+
+export HAVEXMLSTATS HAVEJSONSTATS
diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh
index 37eb737eefe..a9e2f2652f4 100755
--- a/bin/tests/system/run.sh
+++ b/bin/tests/system/run.sh
@@ -109,6 +109,39 @@ export CONTROLPORT
 export LOWPORT
 export HIGHPORT
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:`date`"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -166,17 +199,40 @@ then
    ( cd $systest && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-$PERL start.pl --port $PORT $systest
-if [ $? -ne 0 ]; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:`date $dateargs`"
-    exit 1
+status=0
+run=0
+# Run the tests
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
+fi
+
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+	start_servers
+	rm -f "$systest/$test.status"
+	test_status=0
+	(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+	if [ -f "$systest/$test.status" ]; then
+	    echo_i "FAILED"
+	    test_status=$(cat "$systest/$test.status")
+	fi
+	status=$((status+test_status))
+	stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
 fi
 
-# Run the tests
-( cd $systest ; $SHELL tests.sh "$@" )
-status=$?
 
 if $stopservers
 then
diff --git a/configure.ac b/configure.ac
index 9d0bd6d1124..c921c83aac1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -339,6 +339,12 @@ case "$INSTALL" in
 		;;
 esac
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
+
 AC_PROG_CC
 AC_PROG_CC_C99