build: add pytest targets

It enables running pytests in cmake jobs, regardless of underlying build
tool choice (= makes it work with ninja.)

Also:

- drop pytest logic launching `make` and exiting in case of failure.
  Maybe there is a better way and keep this functionality somehow, bind
  it to a command-line option? make it fail softly?

- GHA/linux: invoke pytest via the build, not directly.

- autotools: add missing dummy runtests targets when cross-compiling.

Closes #15034

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
index 314429f31a1089e0c6d4d72cd03addb638b7c168..460d9426e7609557b67809a0e287dfe261ca8c4a 100644 (file)
--- a/.github/workflows/linux.yml
+++ b/.github/workflows/linux.yml
@@ -652,14 +652,17 @@ jobs:
           fi
           make V=1 VERBOSE=1 test-ci
 
-      - if: contains(matrix.build.install_steps, 'pytest')
-        # run for `tests/http` directory, so pytest does not pick up any other
-        # packages we might have built here
-        run: pytest -v tests/http
-        name: 'run pytest'
+      - name: 'run pytest'
+        if: contains(matrix.build.install_steps, 'pytest')
         env:
           TFLAGS: "${{ matrix.build.tflags }}"
           CURL_CI: github
+        run: |
+          if [ -n '${{ matrix.build.generate }}' ]; then
+            cmake --build . --verbose --target curl-pytest-ci
+          else
+            make V=1 pytest-ci
+          fi
 
       - run: ${{ matrix.build.make-prefix }} make V=1 examples
         if: ${{ matrix.build.configure }}

diff --git a/Makefile.am b/Makefile.am
index 6e477f1a331097e5088ce01ba342378f74f291b3..be3f520bba64f5ecb77aebb1f8e7b202d45e2e29 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -118,7 +118,13 @@ check: test examples check-docs
 
 if CROSSCOMPILING
 test-full: test
+test-nonflaky: test
 test-torture: test
+test-event: test
+test-am: test
+test-ci: test
+pytest: test
+pytest-ci: test
 
 test:
        @echo "NOTICE: we can't run the tests when cross-compiling!"
@@ -146,6 +152,12 @@ test-am:
 test-ci:
        @(cd tests; $(MAKE) all ci-test)
 
+pytest:
+       @(cd tests; $(MAKE) all default-pytest)
+
+pytest-ci:
+       @(cd tests; $(MAKE) all ci-pytest)
+
 endif
 
 examples:

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index b42c93df07a7b78ffa2c9ef4b93db5ba99954cb7..fc22e4e164f0ab412541ffca59bfdfaf89612d2c 100644 (file)
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -43,10 +43,10 @@ function(add_runtests _targetname _test_flags)
   if(CURL_TEST_BUNDLES)
     set(_test_flags "${_test_flags} -bundle")
   endif()
-  unset(_depends)
   # Skip walking through dependent targets before running tests in CI.
   # This avoids: GNU Make doing a slow re-evaluation of all targets and
   # skipping them, MSBuild doing a re-evaluation, and actually rebuilding them.
+  unset(_depends)
   if(NOT _targetname STREQUAL "test-ci")
     set(_depends "testdeps")
   endif()
@@ -65,6 +65,20 @@ function(add_runtests _targetname _test_flags)
   )
 endfunction()
 
+function(add_pytest _targetname _test_flags)
+  unset(_depends)
+  if(NOT _targetname STREQUAL "pytest-ci")
+    set(_depends "test-http-clients")
+  endif()
+  string(REPLACE " " ";" _test_flags_list "${_test_flags}")
+  add_custom_target(${_targetname}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND pytest ${_test_flags_list} "${CMAKE_CURRENT_SOURCE_DIR}/http"
+    DEPENDS "${_depends}"
+    VERBATIM USES_TERMINAL
+  )
+endfunction()
+
 # Create configurehelp.pm, used by tests needing to run the C preprocessor.
 if(MSVC OR CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
   set(CURL_CPP "\"${CMAKE_C_COMPILER}\" -E")
@@ -94,3 +108,6 @@ add_runtests(test-nonflaky  "-a -p ~flaky ~timing-dependent")
 add_runtests(test-ci        "-a -p ~flaky ~timing-dependent -r -rm -j2")
 add_runtests(test-torture   "-a -t -j2")
 add_runtests(test-event     "-a -e")
+
+add_pytest(curl-pytest      "")
+add_pytest(curl-pytest-ci   "-v")

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 250ad3cabb419068323880a02242cae8cc1eb4e4..9638580e53e45e84213912862dde089e75b87408 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -122,6 +122,7 @@ TEST_COMMON =
 
 if CROSSCOMPILING
 TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"
+PYTEST = $(TEST)
 else # if not cross-compiling:
 if USE_TEST_BUNDLES
 TEST_COMMON += -bundle
@@ -140,6 +141,8 @@ TEST_NF = -a -p ~flaky ~timing-dependent
 
 # special CI target derived from nonflaky with CI-specific flags
 TEST_CI = $(TEST_NF) -r -rm -j2
+
+PYTEST = pytest
 endif
 
 CD2NROFF = $(top_srcdir)/scripts/cd2nroff $< >$@
@@ -177,6 +180,11 @@ torture-test: perlcheck all
 event-test: perlcheck all
        $(TEST) $(TEST_E) $(TFLAGS)
 
+default-pytest: ci-pytest
+
+ci-pytest: all
+       srcdir=$(srcdir) $(PYTEST) -v $(srcdir)/http
+
 checksrc:
        (cd libtest && $(MAKE) checksrc)
        (cd unit && $(MAKE) checksrc)

diff --git a/tests/http/clients/CMakeLists.txt b/tests/http/clients/CMakeLists.txt
index 44af35ed3cb5b3d4e282466eb239a98c5ea674ad..fd8994d70a119280602513e5ce73a18f88bb6743 100644 (file)
--- a/tests/http/clients/CMakeLists.txt
+++ b/tests/http/clients/CMakeLists.txt
@@ -22,6 +22,8 @@
 #
 ###########################################################################
 
+add_custom_target(test-http-clients)
+
 # Get 'check_PROGRAMS' variable
 transform_makefile_inc("Makefile.inc" "${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake")
 include("${CMAKE_CURRENT_BINARY_DIR}/Makefile.inc.cmake")
@@ -30,6 +32,7 @@ foreach(_target IN LISTS check_PROGRAMS)
   set(_target_name "curlt-client-${_target}")
   add_executable(${_target_name} EXCLUDE_FROM_ALL "${_target}.c")
   add_dependencies(testdeps ${_target_name})
+  add_dependencies(test-http-clients ${_target_name})
   target_include_directories(${_target_name} PRIVATE
     "${CURL_BINARY_DIR}/lib"  # for "curl_config.h"
     "${CURL_SOURCE_DIR}/lib"  # for "curl_setup.h"

diff --git a/tests/http/conftest.py b/tests/http/conftest.py
index c5ac74544fc99e0465ea0828d5899c3ad2d4936e..44dced022471fe4f1adae1b26948cab3eb1a4260 100644 (file)
--- a/tests/http/conftest.py
+++ b/tests/http/conftest.py
@@ -73,8 +73,6 @@ def env(pytestconfig) -> Env:
         pytest.skip(env.incomplete_reason())
 
     env.setup()
-    if not env.make_clients():
-        pytest.exit(1)
     return env
 
 @pytest.fixture(scope="package", autouse=True)

diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
index be15ea4c62e7aca1ef6e3c871416067a1d8fa400..434cb4051825902bd9704777d79628e41d2b8183 100644 (file)
--- a/tests/http/testenv/env.py
+++ b/tests/http/testenv/env.py
@@ -579,12 +579,3 @@ class Env:
                 i = int(fsize / line_length) + 1
                 fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n")
         return fpath
-
-    def make_clients(self):
-        client_dir = os.path.join(self.project_dir, 'tests/http/clients')
-        p = subprocess.run(['make'], capture_output=True, text=True,
-                           cwd=client_dir)
-        if p.returncode != 0:
-            pytest.exit(f"`make`in {client_dir} failed:\n{p.stderr}")
-            return False
-        return True