Ensure system test programs and deps are compiled

Some system tests require extra programs and/or dependencies to be
compiled first. This is done via `make check` with the automake
framework when using check_* variables such as check_PROGRAMS.

To avoid running any tests via the automake framework, set the TESTS
env variable to empty string and utilize `make -e check` to override
default Makefile variables with environment ones. This ensures automake
will only compile the needed dependencies without running any tests.

Additional consideration needs to be taken for xdist. The compilation
command should be called just once before any tests are executed. To
achieve that, use the pytest_configure() hook and check that the
PYTEST_XDIST_WORKER env variable isn't set -- if it is, it indicates
we're in the spawned xdist worker and the compilation was already done
by the main pytest process that spawned the workers.

This is mostly done to have on-par functionality with legacy test
framework. In the future, we should get rid of the need to run "empty"
make -e check and perhaps compile test-stuff by default.

diff --git a/bin/tests/system/conftest.py b/bin/tests/system/conftest.py
index f77a1b109718b9e64774374e0f274ba1848b7cb9..69d40837fd11cc726ca2ba47c4921b007e1f8695 100644 (file)
--- a/bin/tests/system/conftest.py
+++ b/bin/tests/system/conftest.py
@@ -56,6 +56,7 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
 
     # ----------------------- Globals definition -----------------------------
 
+    XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "")
     FILE_DIR = os.path.abspath(Path(__file__).parent)
     ENV_RE = re.compile("([^=]+)=(.*)")
 
@@ -90,3 +91,31 @@ if os.getenv("LEGACY_TEST_RUNNER", "0") == "0":
     CONF_ENV = get_env(". ./conf.sh && env")
     os.environ.update(CONF_ENV)
     logging.debug("conf.sh env: %s", CONF_ENV)
+
+    # --------------------------- pytest hooks -------------------------------
+
+    def pytest_configure():
+        # Ensure this hook only runs on the main pytest instance if xdist is
+        # used to spawn other workers.
+        if not XDIST_WORKER:
+            logging.debug("compiling required files")
+            env = os.environ.copy()
+            env["TESTS"] = ""  # disable automake test framework - compile-only
+            try:
+                # FUTURE: Remove the need to run this compilation command
+                # before executing tests. Currently it's only here to have
+                # on-par functionality with the legacy test framework.
+                proc = subprocess.run(
+                    "make -e check",
+                    shell=True,
+                    check=True,
+                    cwd=FILE_DIR,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    env=env,
+                )
+            except subprocess.CalledProcessError as exc:
+                logging.debug(exc.stdout)
+                logging.error("failed to compile test files: %s", exc)
+                raise exc
+            logging.debug(proc.stdout)