From: Mostyn Bramley-Moore <mostyn@antipode.se>
Date: Mon, 9 Dec 2024 02:07:49 +0000 (+0100)
Subject: Report skipped tests to cmake/ctest (#2429)
X-Git-Tag: v3.8.0~111
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8ac45a813e78fe6f2f8c40f0219f106a7ff9a28f;p=thirdparty%2Flibarchive.git

Report skipped tests to cmake/ctest (#2429)

This plumbing is required for cmake/ctest to recognise and report
skipped tests.

Now skipped tests in cmake ci jobs are reported like so:
```
          Start   7: libarchive_test_acl_platform_posix1e_read
    7/785 Test   #7: libarchive_test_acl_platform_posix1e_read ................................***Skipped   0.02 sec
```

And there is a list of skipped tests shown at the end of the test run.
---

diff --git a/cat/test/CMakeLists.txt b/cat/test/CMakeLists.txt
index 4a9a0e1ec..e76b47608 100644
--- a/cat/test/CMakeLists.txt
+++ b/cat/test/CMakeLists.txt
@@ -64,7 +64,9 @@ IF(ENABLE_CAT AND ENABLE_TEST)
       COMMAND bsdcat_test -vv
                           -p $<TARGET_FILE:bsdcat>
                           -r ${CMAKE_CURRENT_SOURCE_DIR}
+                          -s
                           ${_testname})
+    SET_TESTS_PROPERTIES(bsdcat_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)
diff --git a/cpio/test/CMakeLists.txt b/cpio/test/CMakeLists.txt
index a3e912fb0..2c530661c 100644
--- a/cpio/test/CMakeLists.txt
+++ b/cpio/test/CMakeLists.txt
@@ -97,7 +97,9 @@ IF(ENABLE_CPIO AND ENABLE_TEST)
       COMMAND bsdcpio_test -vv
                            -p $<TARGET_FILE:bsdcpio>
                            -r ${CMAKE_CURRENT_SOURCE_DIR}
+                           -s
                            ${_testname})
+    SET_TESTS_PROPERTIES(bsdcpio_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)
diff --git a/libarchive/test/CMakeLists.txt b/libarchive/test/CMakeLists.txt
index c8133e65f..77f015099 100644
--- a/libarchive/test/CMakeLists.txt
+++ b/libarchive/test/CMakeLists.txt
@@ -331,7 +331,9 @@ IF(ENABLE_TEST)
       NAME libarchive_${_testname}
       COMMAND libarchive_test -vv
                               -r ${CMAKE_CURRENT_SOURCE_DIR}
+                              -s
                               ${_testname})
+    SET_TESTS_PROPERTIES(libarchive_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)
diff --git a/tar/test/CMakeLists.txt b/tar/test/CMakeLists.txt
index 6a99b656b..53949656b 100644
--- a/tar/test/CMakeLists.txt
+++ b/tar/test/CMakeLists.txt
@@ -115,7 +115,9 @@ IF(ENABLE_TAR AND ENABLE_TEST)
       COMMAND bsdtar_test -vv
                           -p $<TARGET_FILE:bsdtar>
                           -r ${CMAKE_CURRENT_SOURCE_DIR}
+                          -s
                           ${_testname})
+    SET_TESTS_PROPERTIES(bsdtar_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)
diff --git a/test_utils/test_main.c b/test_utils/test_main.c
index 76fcd7940..128a11def 100644
--- a/test_utils/test_main.c
+++ b/test_utils/test_main.c
@@ -370,6 +370,8 @@ invalid_parameter_handler(const wchar_t * expression,
 static int dump_on_failure = 0;
 /* Default is to remove temp dirs and log data for successful tests. */
 static int keep_temp_files = 0;
+/* Default is to only return a failure code (1) if there were test failures. If enabled, exit with code 2 if there were no failures, but some tests were skipped. */
+static int fail_if_tests_skipped = 0;
 /* Default is to run the specified tests once and report errors. */
 static int until_failure = 0;
 /* Default is to just report pass/fail for each test. */
@@ -3672,7 +3674,8 @@ usage(const char *program)
 	printf("  -q  Quiet.\n");
 	printf("  -r <dir>   Path to dir containing reference files.\n");
 	printf("      Default: Current directory.\n");
-	printf("  -u  Keep running specifies tests until one fails.\n");
+	printf("  -s  Exit with code 2 if any tests were skipped.\n");
+	printf("  -u  Keep running specified tests until one fails.\n");
 	printf("  -v  Verbose.\n");
 	printf("Available tests:\n");
 	for (i = 0; i < limit; i++)
@@ -4065,6 +4068,9 @@ main(int argc, char **argv)
 			case 'r':
 				refdir = option_arg;
 				break;
+			case 's':
+				fail_if_tests_skipped = 1;
+				break;
 			case 'u':
 				until_failure++;
 				break;
@@ -4263,5 +4269,9 @@ finish:
 	assertChdir("..");
 	rmdir(tmpdir);
 
-	return (tests_failed ? 1 : 0);
+	if (tests_failed) return 1;
+
+	if (fail_if_tests_skipped == 1 && skips > 0) return 2;
+
+	return 0;
 }
diff --git a/unzip/test/CMakeLists.txt b/unzip/test/CMakeLists.txt
index f269c25f8..789cfb7e4 100644
--- a/unzip/test/CMakeLists.txt
+++ b/unzip/test/CMakeLists.txt
@@ -64,7 +64,9 @@ IF(ENABLE_UNZIP AND ENABLE_TEST)
       COMMAND bsdunzip_test -vv
                            -p $<TARGET_FILE:bsdunzip>
                            -r ${CMAKE_CURRENT_SOURCE_DIR}
+                           -s
                            ${_testname})
+    SET_TESTS_PROPERTIES(bsdunzip_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)