From: Giannis Christodoulou Date: Fri, 7 Aug 2026 16:02:07 +0000 (+0000) Subject: Add --archive option to preserve test results X-Git-Url: http://git.ipfire.org/index.cgi?a=commitdiff_plain;p=thirdparty%2Fapache%2Fhttpd.git Add --archive option to preserve test results * test/conftest.py: Add --archive command-line option. Save per-test conf snapshots via archive_test_conf after each test. Call archive_logs at end of each package. * test/pyhttpd/env.py: Add archive_logs and archive_test_conf to copy logs and per-test confs to the archive folder, organised by httpd version and git/svn revision. Add ignore_files to exclude htdocs, shared confs and sockets from per-module copies. Move shared entries (ca, ca1, ca2, certs, httpd.conf, mime.types, stop.conf) to _shared/. Wipe logs dir at module setup to avoid stale files from previous runs. * test/README.pytest: Document the --archive option, folder structure and what gets archived. Assisted-by: Claude Sonnet 4.6 git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1936981 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/test/README.pytest b/test/README.pytest index e6630ac394..0b5cfd57a4 100644 --- a/test/README.pytest +++ b/test/README.pytest @@ -35,6 +35,50 @@ also raises the error log level of the tested modules. > pytest -vvv -k test_h2_004_01 run the specific test with mod_http2 at log level TRACE2. +There is an option to archive the results across different +modules and httpd versions. The archiving will preserve +error_log and access_log (cumulative for all tests in the module), +and save a separate config file for each test case. +> pytest test/modules/core --archive=/path/to/archive + +If you don't provide any specific module, pytest will execute all of them +and add a new folder per module to the existing structure. +> pytest --archive=/path/to/archive + +Always use --archive= (with =, not a space) to avoid pytest rootdir issues. + +The archive folder is named after the httpd version and the git commit hash +so results from different code versions are kept separate. +Running the same module again with the same commit updates its folder in place. + +What gets archived: +- error_log and access_log for the module run +- per-test configs: test.conf snapshot after each test +- module config: modules.conf +- shared configs and infrastructure in _shared/ to avoid duplication + +Example archive structure with 2 httpd versions: + ├── /path/to/archive + │ ├── 2.4.65-a3f8c1d + │ │ ├── _shared + │ │ │ ├── conf + │ │ │ │ ├── httpd.conf + │ │ │ │ ├── mime.types + │ │ │ │ └── stop.conf + │ │ │ ├── ca + │ │ ├── core + │ │ │ ├── conf + │ │ │ │ ├── modules.conf + │ │ │ │ ├── test_core_001_01.conf + │ │ │ └── logs + │ │ │ ├── error_log + │ │ │ └── access_log + │ │ ├── http1 + │ │ └── http2 + │ └── 2.4.66-b7d2e4f + │ ├── _shared + │ └── core + By default, test cases will configure httpd with mpm_event. You can change that with the invocation: > MPM=worker pytest test/modules/http2 diff --git a/test/conftest.py b/test/conftest.py index 390a33bec5..1fa0e751e0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,6 @@ import sys import os +import warnings import pytest @@ -15,6 +16,9 @@ def pytest_addoption(parser): parser.addoption("--repeat", action="store", type=int, default=1, help='Number of times to repeat each test') parser.addoption("--all", action="store_true") + parser.addoption("--archive", action="store", default=None, + help='Archive the server dir after each test package ' + 'to the specified folder') def pytest_generate_tests(metafunc): @@ -29,6 +33,12 @@ def _function_scope(env, request): env.set_current_test_name(request.node.name) yield env.check_error_log() + archive_dir = request.config.getoption("--archive") + if archive_dir: + fspath = str(request.fspath) + if 'modules/' in fspath: + package_name = fspath.split('modules/')[1].split('/')[0] + env.archive_test_conf(request.node.name, package_name, archive_dir) env.set_current_test_name(None) @@ -39,9 +49,17 @@ def _module_scope(env): @pytest.fixture(autouse=True, scope="package") -def _package_scope(env): +def _package_scope(env, request): env.httpd_error_log.clear_ignored_matches() env.httpd_error_log.clear_ignored_lognos() yield assert env.apache_stop() == 0 env.check_error_log() + archive_dir = request.config.getoption("--archive") + if archive_dir == "": + warnings.warn("--archive option was empty, skipping archiving") + if archive_dir: + fspath = str(request.fspath) + parts = fspath.split('modules/') + package_name = parts[1].split('/')[0] + env.archive_logs(package_name, archive_dir) diff --git a/test/pyhttpd/env.py b/test/pyhttpd/env.py index a3e020dd99..0ecc31b96d 100644 --- a/test/pyhttpd/env.py +++ b/test/pyhttpd/env.py @@ -99,8 +99,10 @@ class HttpdTestSetup: def _make_dirs(self): if not os.path.exists(self.env.gen_dir): os.makedirs(self.env.gen_dir) - if not os.path.exists(self.env.server_logs_dir): - os.makedirs(self.env.server_logs_dir) + # wipe logs from any previous module run to avoid stale files in the archive + if os.path.isdir(self.env.server_logs_dir): + shutil.rmtree(self.env.server_logs_dir) + os.makedirs(self.env.server_logs_dir) def _make_conf(self): # remove anything from another run/test suite @@ -224,6 +226,9 @@ class HttpdTestSetup: class HttpdTestEnv: LIBEXEC_DIR = None + # entries shared across all modules, archived once in _shared + SHARED_SERVER_ENTRIES = {'ca', 'ca1', 'ca2', 'md', 'acme-ca.pem', + 'eab.json'} @staticmethod def has_tool(name: str) -> bool: @@ -360,6 +365,81 @@ class HttpdTestEnv: f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\ "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings)) + # combines httpd version & revision to identify the archive folder + def _archive_version(self): + if not hasattr(self, '_cached_archive_version'): + version = self.get_httpd_version() + r = subprocess.run(['git', 'rev-parse', '--short', 'HEAD'], + capture_output=True, text=True, + cwd=self._our_dir) + if r.returncode == 0: + version = f"{version}-{r.stdout.strip()}" + else: + r = subprocess.run(['svn', 'info', '--show-item', 'revision'], + capture_output=True, text=True, + cwd=self._our_dir) + + if r.returncode == 0: + version = f"{version}-r{r.stdout.strip()}" + self._cached_archive_version = version + return self._cached_archive_version + + def archive_logs(self, package_name, archive_dir): + version = self._archive_version() + dest = os.path.join(archive_dir, version, package_name) + os.makedirs(dest, exist_ok=True) + + # merge into existing dest to preserve per-test confs + shutil.copytree(self._server_dir, dest, ignore=self.ignore_files, + dirs_exist_ok=True) + + shared_dest = os.path.join(archive_dir, version, '_shared') + + if os.path.isdir(shared_dest): + shutil.rmtree(shared_dest) + os.makedirs(shared_dest) + + for entry in self.SHARED_SERVER_ENTRIES: + src = os.path.join(self._server_dir, entry) + entry_dest = os.path.join(shared_dest, entry) + if os.path.isdir(src): + shutil.copytree(src, entry_dest) + elif os.path.isfile(src): + shutil.copy2(src, entry_dest) + + shared_conf_dest = os.path.join(shared_dest, 'conf') + os.makedirs(shared_conf_dest) + + # copy them once + for f in ['httpd.conf', 'mime.types', 'stop.conf']: + src = os.path.join(self._server_conf_dir, f) + if os.path.isfile(src): + shutil.copy2(src, shared_conf_dest) + + def archive_test_conf(self, test_name, package_name, archive_dir): + version = self._archive_version() + dest = os.path.join(archive_dir, version, package_name, 'conf') + if not os.path.isdir(dest): + os.makedirs(dest) + test_conf = os.path.join(self._server_conf_dir, 'test.conf') + if os.path.isfile(test_conf): + safe_name = test_name.replace('/', '_').replace('\\', '_') + dest_file = os.path.join(dest, f"{safe_name}.conf") + shutil.copy(test_conf, dest_file) + + # return files to ignore + def ignore_files(self, d, entries): + ignored = [e for e in entries if + e.endswith('.sock') or e in self.SHARED_SERVER_ENTRIES] + # shared confs goes to _shared + # test.conf is captured per-test + if os.path.basename(d) == 'conf': + ignored += ['httpd.conf', 'mime.types', 'stop.conf', 'test.conf'] + # htdocs are static test fixtures not useful in the archive + if os.path.basename(d) == os.path.basename(self._server_dir): + ignored += ['htdocs'] + return ignored + @property def curl(self) -> str: return self._curl