Report skipped tests to cmake/ctest (#2429)

This plumbing is required for cmake/ctest to recognise and report
skipped tests.

Now skipped tests in cmake ci jobs are reported like so:
```
          Start   7: libarchive_test_acl_platform_posix1e_read
    7/785 Test   #7: libarchive_test_acl_platform_posix1e_read ................................***Skipped   0.02 sec
```

And there is a list of skipped tests shown at the end of the test run.

diff --git a/cat/test/CMakeLists.txt b/cat/test/CMakeLists.txt
index 4a9a0e1ecf2538483367d86b06a6b2051f9b7b02..e76b47608728cc8b331b438820c2dbcf33778cb1 100644 (file)
--- a/cat/test/CMakeLists.txt
+++ b/cat/test/CMakeLists.txt
@@ -64,7 +64,9 @@ IF(ENABLE_CAT AND ENABLE_TEST)
       COMMAND bsdcat_test -vv
                           -p $<TARGET_FILE:bsdcat>
                           -r ${CMAKE_CURRENT_SOURCE_DIR}
+                          -s
                           ${_testname})
+    SET_TESTS_PROPERTIES(bsdcat_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)

diff --git a/cpio/test/CMakeLists.txt b/cpio/test/CMakeLists.txt
index a3e912fb0f653169a2227040b79dbfdc97aba677..2c530661cfa788fb2a3c7ab41f12783dbb7cfe88 100644 (file)
--- a/cpio/test/CMakeLists.txt
+++ b/cpio/test/CMakeLists.txt
@@ -97,7 +97,9 @@ IF(ENABLE_CPIO AND ENABLE_TEST)
       COMMAND bsdcpio_test -vv
                            -p $<TARGET_FILE:bsdcpio>
                            -r ${CMAKE_CURRENT_SOURCE_DIR}
+                           -s
                            ${_testname})
+    SET_TESTS_PROPERTIES(bsdcpio_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)

diff --git a/libarchive/test/CMakeLists.txt b/libarchive/test/CMakeLists.txt
index c8133e65ff04803105a217e0e96ac9ab741629d4..77f01509986dbb4c5b6e2d03551f9d6db8e0e396 100644 (file)
--- a/libarchive/test/CMakeLists.txt
+++ b/libarchive/test/CMakeLists.txt
@@ -331,7 +331,9 @@ IF(ENABLE_TEST)
       NAME libarchive_${_testname}
       COMMAND libarchive_test -vv
                               -r ${CMAKE_CURRENT_SOURCE_DIR}
+                              -s
                               ${_testname})
+    SET_TESTS_PROPERTIES(libarchive_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)

diff --git a/tar/test/CMakeLists.txt b/tar/test/CMakeLists.txt
index 6a99b656b423166575f00645c719637a436aff03..53949656b754b58b43d5137be6547d6b918df2b8 100644 (file)
--- a/tar/test/CMakeLists.txt
+++ b/tar/test/CMakeLists.txt
@@ -115,7 +115,9 @@ IF(ENABLE_TAR AND ENABLE_TEST)
       COMMAND bsdtar_test -vv
                           -p $<TARGET_FILE:bsdtar>
                           -r ${CMAKE_CURRENT_SOURCE_DIR}
+                          -s
                           ${_testname})
+    SET_TESTS_PROPERTIES(bsdtar_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)

diff --git a/test_utils/test_main.c b/test_utils/test_main.c
index 76fcd79406ff4456ae485b0c1b132512600887a7..128a11def404a95bdb4b51a234d28aa799df87c0 100644 (file)
--- a/test_utils/test_main.c
+++ b/test_utils/test_main.c
@@ -370,6 +370,8 @@ invalid_parameter_handler(const wchar_t * expression,
 static int dump_on_failure = 0;
 /* Default is to remove temp dirs and log data for successful tests. */
 static int keep_temp_files = 0;
+/* Default is to only return a failure code (1) if there were test failures. If enabled, exit with code 2 if there were no failures, but some tests were skipped. */
+static int fail_if_tests_skipped = 0;
 /* Default is to run the specified tests once and report errors. */
 static int until_failure = 0;
 /* Default is to just report pass/fail for each test. */
@@ -3672,7 +3674,8 @@ usage(const char *program)
        printf("  -q  Quiet.\n");
        printf("  -r <dir>   Path to dir containing reference files.\n");
        printf("      Default: Current directory.\n");
-       printf("  -u  Keep running specifies tests until one fails.\n");
+       printf("  -s  Exit with code 2 if any tests were skipped.\n");
+       printf("  -u  Keep running specified tests until one fails.\n");
        printf("  -v  Verbose.\n");
        printf("Available tests:\n");
        for (i = 0; i < limit; i++)
@@ -4065,6 +4068,9 @@ main(int argc, char **argv)
                        case 'r':
                                refdir = option_arg;
                                break;
+                       case 's':
+                               fail_if_tests_skipped = 1;
+                               break;
                        case 'u':
                                until_failure++;
                                break;
@@ -4263,5 +4269,9 @@ finish:
        assertChdir("..");
        rmdir(tmpdir);
 
-       return (tests_failed ? 1 : 0);
+       if (tests_failed) return 1;
+
+       if (fail_if_tests_skipped == 1 && skips > 0) return 2;
+
+       return 0;
 }

diff --git a/unzip/test/CMakeLists.txt b/unzip/test/CMakeLists.txt
index f269c25f85e6d88ccd356703db268a28f6ec3fbc..789cfb7e492e1f375bdb6902c37fa3d290036bac 100644 (file)
--- a/unzip/test/CMakeLists.txt
+++ b/unzip/test/CMakeLists.txt
@@ -64,7 +64,9 @@ IF(ENABLE_UNZIP AND ENABLE_TEST)
       COMMAND bsdunzip_test -vv
                            -p $<TARGET_FILE:bsdunzip>
                            -r ${CMAKE_CURRENT_SOURCE_DIR}
+                           -s
                            ${_testname})
+    SET_TESTS_PROPERTIES(bsdunzip_${_testname} PROPERTIES SKIP_RETURN_CODE 2)
   ENDMACRO (DEFINE_TEST _testname)
 
   INCLUDE(${CMAKE_CURRENT_BINARY_DIR}/list.h)