Add pytest fixture for checking test artifacts

Prior to introducing the pytest runner, clean.sh files were used as a
list of files that the test is expected to leave around as artifacts and
check that no extra files were created.

With the pytest runner, those scripts are no longer used, but the
ability to detect extraneous files is still useful. Add a new
"extra_artifacts" mark which can be used for the same purpose.

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 7ffb2319b8482e13bd570e5fd564250b19d14b4b..e4fd7c76258c524759767356be744f194ad3b524 100644 (file)
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -10,6 +10,7 @@
 # information regarding copyright ownership.
 
 from functools import partial
+import filecmp
 import os
 from pathlib import Path
 import re
@@ -297,7 +298,28 @@ def logger(request, system_test_name):
 
 
 @pytest.fixture(scope="module")
-def system_test_dir(request, system_test_name):
+def expected_artifacts(request):
+    common_artifacts = [
+        "ns*/named.run",
+        "ns*/named.run.prev",
+        "ns*/named.conf",
+        "ns*/named.memstats",
+        "pytest.log.txt",
+    ]
+
+    try:
+        test_specific_artifacts = request.node.get_closest_marker("extra_artifacts")
+    except AttributeError:
+        return None
+
+    if test_specific_artifacts:
+        return common_artifacts + test_specific_artifacts.args[0]
+
+    return common_artifacts
+
+
+@pytest.fixture(scope="module")
+def system_test_dir(request, system_test_name, expected_artifacts):
     """
     Temporary directory for executing the test.
 
@@ -345,6 +367,38 @@ def system_test_dir(request, system_test_name):
         except FileNotFoundError:
             pass
 
+    def check_artifacts(source_dir, run_dir):
+        def check_artifacts_recursive(dcmp):
+            def artifact_expected(path, expected):
+                for glob in expected:
+                    if path.match(glob):
+                        return True
+                return False
+
+            # test must not remove any Git-tracked file, ignore libtool and gcov artifacts
+            for name in dcmp.left_only:
+                assert name.startswith("lt-") or name.endswith(".gcda")
+            assert not dcmp.diff_files, "test must not modify any Git-tracked file"
+
+            dir_path = Path(dcmp.left).relative_to(source_dir)
+            for name in dcmp.right_only:
+                file = dir_path / Path(name)
+                if not artifact_expected(file, expected_artifacts):
+                    unexpected_files.append(str(file))
+            for subdir in dcmp.subdirs.values():
+                check_artifacts_recursive(subdir)
+
+        if expected_artifacts is None:  # skip the check if artifact list is unavailable
+            return
+
+        unexpected_files = []
+        dcmp = filecmp.dircmp(source_dir, run_dir)
+        check_artifacts_recursive(dcmp)
+
+        assert (
+            not unexpected_files
+        ), f"Unexpected files found in test directory: {unexpected_files}"
+
     # Create a temporary directory with a copy of the original system test dir contents
     system_test_root = Path(os.environ["builddir"])
     testdir = Path(
@@ -374,6 +428,9 @@ def system_test_dir(request, system_test_name):
 
         result = get_test_result()
 
+        if result == "passed":
+            check_artifacts(system_test_root / system_test_name, testdir)
+
         # Clean temporary dir unless it should be kept
         keep = False
         if request.config.getoption("--noclean"):

diff --git a/bin/tests/system/pytest.ini b/bin/tests/system/pytest.ini
index 766b3dad2ea73675a35d7c4fcc457996080bfbb1..5574b3e61150f827b425197a20bb6e726ad54c17 100644 (file)
--- a/bin/tests/system/pytest.ini
+++ b/bin/tests/system/pytest.ini
@@ -21,3 +21,4 @@ junit_log_passing_tests = 0
 markers =
     requires_zones_loaded: ensures the test does not start until the specified named instances load all configured zones
     algorithm_set: use to select desired algorithms from isctest/vars/algorithms.py
+    extra_artifacts: list of files (globs) that are expected to appear in the test directory after the test is run