Create symlinks to test artifacts for pytest runner

While temporary directories are useful for test execution to keep
everything clean, they are difficult to work with manually. Create a
symlink for each test artifact directory with a stable and predictable
path. The symlink always either points to the latest artifacts, or is
missing in case the last run succeeded.

Ensure these symlinked directories aren't detected as test suites by the
pytest runner.

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index 766ed6a8000e5bd2857b6f8f06cc41ce9ba29a1e..0488948e4adfb392dd8f96b279543bfacbbffd70 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -2,6 +2,7 @@
 .hypothesis
 .mypy_cache
 __pycache__
+_last_test_run
 dig.out*
 rndc.out*
 nsupdate.out*

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 29799266e40c6575e79b12e45da743cea7320f39..fd7510827fbe3290b914513eea438454b86311e3 100644 (file)
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -104,6 +104,9 @@ else:
     ]
     PRIORITY_TESTS_RE = re.compile("|".join(PRIORITY_TESTS))
     CONFTEST_LOGGER = logging.getLogger("conftest")
+    SYSTEM_TEST_DIR_GIT_PATH = "bin/tests/system"
+    SYSTEM_TEST_NAME_RE = re.compile(f"{SYSTEM_TEST_DIR_GIT_PATH}" + r"/([^/]+)")
+    SYMLINK_REPLACEMENT_RE = re.compile(r"/tests(_sh(?=_))?(.*)\.py")
 
     # ---------------------- Module initialization ---------------------------
 
@@ -227,8 +230,16 @@ else:
         # bin/tests/system. These temporary directories contain all files
         # needed for the system tests - including tests_*.py files. Make sure to
         # ignore these during test collection phase. Otherwise, test artifacts
-        # from previous runs could mess with the runner.
-        return "_tmp_" in str(path)
+        # from previous runs could mess with the runner. Also ignore the
+        # convenience symlinks to those test directories. In both of those
+        # cases, the system test name (directory) contains an underscore, which
+        # is otherwise and invalid character for a system test name.
+        match = SYSTEM_TEST_NAME_RE.search(str(path))
+        if match is None:
+            CONFTEST_LOGGER.warning("unexpected test path: %s (ignored)", path)
+            return True
+        system_test_name = match.groups()[0]
+        return "_" in system_test_name
 
     def pytest_collection_modifyitems(items):
         """Schedule long-running tests first to get more benefit from parallelism."""
@@ -345,8 +356,8 @@ else:
         """Dictionary containing environment variables for the test."""
         env = os.environ.copy()
         env.update(ports)
-        env["builddir"] = f"{env['TOP_BUILDDIR']}/bin/tests/system"
-        env["srcdir"] = f"{env['TOP_SRCDIR']}/bin/tests/system"
+        env["builddir"] = f"{env['TOP_BUILDDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}"
+        env["srcdir"] = f"{env['TOP_SRCDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}"
         return env
 
     @pytest.fixture(scope="module")
@@ -409,14 +420,26 @@ else:
             assert all(res.outcome == "passed" for res in test_results.values())
             return "passed"
 
+        def unlink(path):
+            try:
+                path.unlink()  # missing_ok=True isn't available on Python 3.6
+            except FileNotFoundError:
+                pass
+
         # Create a temporary directory with a copy of the original system test dir contents
-        system_test_root = Path(f"{env['TOP_BUILDDIR']}/bin/tests/system")
+        system_test_root = Path(f"{env['TOP_BUILDDIR']}/{SYSTEM_TEST_DIR_GIT_PATH}")
         testdir = Path(
             tempfile.mkdtemp(prefix=f"{system_test_name}_tmp_", dir=system_test_root)
         )
         shutil.rmtree(testdir)
         shutil.copytree(system_test_root / system_test_name, testdir)
 
+        # Create a convenience symlink with a stable and predictable name
+        module_name = SYMLINK_REPLACEMENT_RE.sub(r"\2", request.node.name)
+        symlink_dst = system_test_root / module_name
+        unlink(symlink_dst)
+        symlink_dst.symlink_to(os.path.relpath(testdir, start=system_test_root))
+
         # Configure logger to write to a file inside the temporary test directory
         mlogger.handlers.clear()
         mlogger.setLevel(logging.DEBUG)
@@ -451,6 +474,7 @@ else:
                 handler.flush()
                 handler.close()
                 shutil.rmtree(testdir)
+                unlink(symlink_dst)
 
     def _run_script(  # pylint: disable=too-many-arguments
         env,