From: Ondřej Surý <ondrej@sury.org>
Date: Fri, 27 Mar 2020 12:23:24 +0000 (+0100)
Subject: Add a pytest runner to run.sh
X-Git-Tag: v9.17.2~118^2
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=cf5105939c1c1802e69cc63a39e9d71b6e5f67e1;p=thirdparty%2Fbind9.git

Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.
---

diff --git a/.gitignore b/.gitignore
index 71607aa297c..742df701d09 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,7 @@
 *_test
 *.ipch # vscode/intellisense precompiled header
 *~
+__pycache__/
 .ccache/
 .cproject
 .deps/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0662fc1de3..65a3fc0f8fb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -199,6 +199,7 @@ stages:
   before_script:
     - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
     - test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - *configure
     - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
@@ -256,6 +257,7 @@ stages:
   before_script:
     - *setup_interfaces
     - *setup_softhsm
+    - pip3 install pytest requests || pip install pytest requests || true
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1
@@ -441,6 +443,7 @@ pylint:
   before_script:
     - pip3 install pylint
     - PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
+    - pip3 install pytest requests || pip install pytest requests
   script:
     - *configure
     - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')
diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 0c66e8d2cc9..d7c9c7b1bc3 100644
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -1,3 +1,4 @@
+__pycache__
 dig.out*
 rndc.out*
 nsupdate.out*
diff --git a/bin/tests/system/conf.sh.in b/bin/tests/system/conf.sh.in
index 85f8df69e4d..4088fb4411c 100644
--- a/bin/tests/system/conf.sh.in
+++ b/bin/tests/system/conf.sh.in
@@ -112,6 +112,7 @@ PERL=$(command -v "@PERL@")
 PSSUSPEND=
 
 PYTHON=$(command -v "@PYTHON@")
+PYTEST=@PYTEST@
 
 #
 # Determine if we support various optional features.
@@ -127,3 +128,5 @@ HAVEZLIB=${ZLIB_LIBS:+1}
 LMDB_LIBS="@LMDB_LIBS@"
 NZD=${LMDB_LIBS:+1}
 CRYPTO=@CRYPTO@
+
+export HAVEXMLSTATS HAVEJSONSTATS
diff --git a/bin/tests/system/run.sh.in b/bin/tests/system/run.sh.in
index 5e4095ea67d..4e8f5cc6b90 100644
--- a/bin/tests/system/run.sh.in
+++ b/bin/tests/system/run.sh.in
@@ -108,6 +108,39 @@ fi
 # true, the last digit of EXTRAPORTn is "n".
 eval "$(${srcdir}/get_ports.sh -p "$baseport")"
 
+restart=false
+
+start_servers_failed() {
+    echoinfo "I:$systest:starting servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+start_servers() {
+    echoinfo "I:$systest:starting servers"
+    if $restart; then
+        $PERL start.pl --restart --port "$PORT" "$systest" || start_fail
+    else
+        restart=true
+        $PERL start.pl --port "$PORT" "$systest" || start_fail
+    fi
+}
+
+stop_servers_failed() {
+    echoinfo "I:$systest:stopping servers failed"
+    echofail "R:$systest:FAIL"
+    echoend  "E:$systest:$(date_with_args)"
+    exit 1
+}
+
+stop_servers() {
+    if $stopservers; then
+        echoinfo "I:$systest:stopping servers"
+        $PERL stop.pl "$systest" || stop_servers_failed
+    fi
+}
+
 echostart "S:$systest:$(date_with_args)"
 echoinfo  "T:$systest:1:A"
 echoinfo  "A:$systest:System test $systest"
@@ -161,16 +194,40 @@ then
    ( cd "${systest}" && $SHELL setup.sh "$@" )
 fi
 
-# Start name servers running
-if ! $PERL ${srcdir}/start.pl --port "$PORT" "$systest"; then
-    echofail "R:$systest:FAIL"
-    echoend  "E:$systest:$(date_with_args)"
-    exit 1
-fi
 
+status=0
+run=0
 # Run the tests
-( cd "${systest}" && $SHELL tests.sh "$@" )
-status=$?
+if [ -r "$systest/tests.sh" ]; then
+    start_servers
+    ( cd "$systest" && $SHELL tests.sh "$@" )
+    status=$?
+    run=$((run+1))
+    stop_servers
+fi
+
+if [ -n "$PYTEST" ]; then
+    run=$((run+1))
+    for test in $(cd "${systest}" && find . -name "tests*.py"); do
+	start_servers
+	rm -f "$systest/$test.status"
+	test_status=0
+	(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
+	if [ -f "$systest/$test.status" ]; then
+	    echo_i "FAILED"
+	    test_status=$(cat "$systest/$test.status")
+	fi
+	status=$((status+test_status))
+	stop_servers
+    done
+else
+    echoinfo "I:$systest:pytest not installed, skipping python tests"
+fi
+
+if [ "$run" -eq "0" ]; then
+    echoinfo "I:$systest:No tests were found and run"
+    status=255
+fi
 
 if $stopservers
 then
@@ -179,11 +236,6 @@ else
     exit $status
 fi
 
-# Shutdown
-$PERL ${srcdir}/stop.pl "$systest"
-
-status=$((status + $?))
-
 if [ $status != 0 ]; then
     echofail "R:$systest:FAIL"
     # Do not clean up - we need the evidence.
diff --git a/configure.ac b/configure.ac
index 8ad62afb270..458bd6cda26 100644
--- a/configure.ac
+++ b/configure.ac
@@ -276,6 +276,11 @@ AM_CONDITIONAL([HAVE_PERLMOD_TIME_HIRES],
 AM_PATH_PYTHON([3.4], [], [:])
 AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
 
+AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
+AS_IF([test -z "$PYTEST"],
+      [AC_MSG_WARN([pytest not found, some system tests will be skipped])])
+AC_SUBST([PYTEST])
+
 AX_PYTHON_MODULE([dns])
 AM_CONDITIONAL([HAVE_PYMOD_DNS], [test "$HAVE_PYMOD_DNS" = "yes"])