From: Tom Krizek <tkrizek@isc.org>
Date: Tue, 22 Nov 2022 09:47:15 +0000 (+0100)
Subject: Use pytest system test runner in CI
X-Git-Tag: v9.19.14~35^2~20
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=4bc2b3be48e23152cdced9517f7b1d2839fde8f6;p=thirdparty%2Fbind9.git

Use pytest system test runner in CI

Replace the legacy system test runner by the pytest system test runner.
Since EL7 and OpenBSD have only ancient versions of pytest / xdist, keep
using the legacy test runner there for now.

Out of tree tests aren't supported by the pytest runner yet. Use the
legacy test runner for that purpose as well.

Use awk to display failures and errors at the end of the log for
convenience, since pytest displays them first, which makes them
difficult to find.
---

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 62f811d0e01..5f388c9f6f0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -253,6 +253,10 @@ stages:
   - PYTHON="$(source bin/tests/system/conf.sh; echo $PYTHON)"
   - test -x "$PYTHON"
 
+.find_pytest: &find_pytest
+  - PYTEST="$(source bin/tests/system/conf.sh; echo $PYTEST)"
+  - test -x "$PYTEST"
+
 .parse_tsan: &parse_tsan
     - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \;
 
@@ -317,6 +321,17 @@ stages:
   before_script:
     - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}"
     - *setup_interfaces
+  script:
+    - *find_pytest
+    - cd bin/tests/system
+    - >
+      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" --dist loadscope | tee pytest.out.txt
+    - '( ! grep -F "grep: warning:" pytest.out.txt )'
+  after_script:
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+
+.system_test_legacy: &system_test_legacy
   script:
     - cd bin/tests/system
     - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1
@@ -336,6 +351,8 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -344,12 +361,15 @@ stages:
   <<: *system_test_common
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
 
 .system_test_tsan: &system_test_tsan_job
   <<: *system_test_common
   after_script:
-    - cat bin/tests/system/test-suite.log
+    - awk '/^=+ FAILURES =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
+    - awk '/^=+ ERRORS =+/{flag=1;next}/^=+.*=+$/{flag=0}flag' bin/tests/system/pytest.out.txt || true
     - find bin/tests/system -name "*dig.*" | xargs grep "error" || true
     - *find_python
     - *parse_tsan
@@ -357,6 +377,8 @@ stages:
       "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml
   artifacts:
     untracked: true
+    exclude:
+      - "**/__pycache__/**/*"
     when: always
     reports:
       junit: junit.xml
@@ -859,6 +881,7 @@ system:gcc:out-of-tree:
       artifacts: true
   <<: *base_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_tags_triggers_web_triggering_rules
 
 unit:gcc:out-of-tree:
@@ -1193,6 +1216,7 @@ clang:openbsd:amd64:
 system:clang:openbsd:amd64:
   <<: *openbsd_amd64_image
   <<: *system_test_job
+  <<: *system_test_legacy
   <<: *api_schedules_triggers_web_triggering_rules
   variables:
     USER: gitlab-runner