Use underscore for system test names

Change the convention for system test directory names to always use an
underscore rather than a hyphen. Names using underscore are valid python
package names and can be used with standard `import` facilities in
python, which allows easier code reuse.

The temporary directories for test execution and their convenience
symlinks have been switched to using hyphens rather than underscores to
keep the pytest collection, filtering and .gitignore working as
expected.

diff --git a/bin/tests/system/.gitignore b/bin/tests/system/.gitignore
index c0f44047acc258eb65397bbbd8ffac51ab6f9d81..a59bb20737038aa00115433ac578ff60e220def7 100644 (file)
--- a/bin/tests/system/.gitignore
+++ b/bin/tests/system/.gitignore
@@ -25,10 +25,8 @@ named.run
 !/isctest/vars/.build_vars/*.in
 !/isctest/vars/.build_vars/meson.build
 
-# Ignore file names with underscore in their name except python or shell files.
+# Ignore file names with a hyphen in their name except C files.
 # This is done to ignore the temporary directories and symlinks created by the
 # pytest runner, which contain underscore in their file names.
-/*_*
-!/*_*.py
-!/*_*.sh
-!/_common
+/*-*
+!/*-*.c

diff --git a/bin/tests/system/README.md b/bin/tests/system/README.md
index dc5f782779cd41409d708db1d5f5b47492c9c1e5..96048c15a5b1f202c2c99bfd17e84c082a132da5 100644 (file)
--- a/bin/tests/system/README.md
+++ b/bin/tests/system/README.md
@@ -123,15 +123,12 @@ Each test module is executed inside a unique temporary directory which contains
 all the artifacts from the test run. If the tests succeed, they are deleted by
 default. To override this behaviour, pass `--noclean` to pytest.
 
-The directory name starts with the system test name, followed by `_tmp_XXXXXX`,
-i.e. `dns64_tmp_r07vei9s` for `dns64` test run. Since this name changes each
+The directory name starts with the system test name, followed by `-tmp-XXXXXX`,
+i.e. `dns64-tmp-r07vei9s` for `dns64` test run. Since this name changes each
 run, a convenience symlink that has a stable name is also created. It points to
-the latest test artifacts directory and has a form of `dns64_sh_dns64`
+the latest test artifacts directory and has a form of `dns64-sh_dns64`
 (depending on the particular test module).
 
-To clean up the temporary directories and symlinks, run `make clean-local` in
-the system test directory.
-
 The following test artifacts are typically available:
 
 - pytest.log.txt: main log file with test output

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index 4e7a1d7b1a15a2eafe3ae7da804e9ce105bcf761..74af6c1b5ea32487cfb7f8e48808d41015c2f80c 100644 (file)
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -54,7 +54,7 @@ PRIORITY_TESTS = [
 ]
 PRIORITY_TESTS_RE = Re("|".join(PRIORITY_TESTS))
 SYSTEM_TEST_NAME_RE = Re(f"{SYSTEM_TEST_DIR_GIT_PATH}" + r"/([^/]+)")
-SYMLINK_REPLACEMENT_RE = Re(r"/tests(_.*)\.py")
+SYMLINK_REPLACEMENT_RE = Re(r"/tests_(.*)\.py")
 
 # ----------------------- Global requirements ----------------------------
 
@@ -84,14 +84,14 @@ def pytest_ignore_collect(collection_path):
     # ignore these during test collection phase. Otherwise, test artifacts
     # from previous runs could mess with the runner. Also ignore the
     # convenience symlinks to those test directories. In both of those
-    # cases, the system test name (directory) contains an underscore, which
+    # cases, the system test name (directory) contains a hyphen, which
     # is otherwise and invalid character for a system test name.
     match = SYSTEM_TEST_NAME_RE.search(str(collection_path))
     if match is None:
         isctest.log.warning("unexpected test path: %s (ignored)", collection_path)
         return True
     system_test_name = match.groups()[0]
-    return "_" in system_test_name
+    return "-" in system_test_name
 
 
 def pytest_collection_modifyitems(items):
@@ -385,14 +385,14 @@ def system_test_dir(request, system_test_name, expected_artifacts):
     # Create a temporary directory with a copy of the original system test dir contents
     system_test_root = Path(os.environ["srcdir"]).resolve()
     testdir = Path(
-        tempfile.mkdtemp(prefix=f"{system_test_name}_tmp_", dir=system_test_root)
+        tempfile.mkdtemp(prefix=f"{system_test_name}-tmp-", dir=system_test_root)
     )
     shutil.rmtree(testdir)
     shutil.copytree(system_test_root / system_test_name, testdir)
     isctest.vars.dirs.set_system_test_name(testdir.name)
 
     # Create a convenience symlink with a stable and predictable name
-    module_name = SYMLINK_REPLACEMENT_RE.sub(r"\1", str(request.node.path))
+    module_name = SYMLINK_REPLACEMENT_RE.sub(r"-\1", str(request.node.path))
     symlink_dst = system_test_root / module_name
     symlink_dst.unlink(missing_ok=True)
     symlink_dst.symlink_to(os.path.relpath(testdir, start=system_test_root))

diff --git a/dangerfile.py b/dangerfile.py
index e2ba7d48fdc9964c60eab2710a9e839348545d05..33f2fbe4fe18f06c3d19b3cffcf57c8e4467f4ef 100644 (file)
--- a/dangerfile.py
+++ b/dangerfile.py
@@ -483,10 +483,10 @@ for testname in testnames:
         or testname == "isctest"
     ):
         continue
-    if "_" in testname:
+    if "-" in testname:
         fail(
-            f"System test directory `{testname}` may not contain an underscore, "
-            "use hyphen instead."
+            f"System test directory `{testname}` may not contain a hyphen, "
+            "use underscore instead."
         )
     if not glob.glob(f"{dirpath}/**/tests_*.py", recursive=True):
         fail(