test-lib: add a --invert-exit-code switch

Add the ability to have those tests that fail return 0, and those
tests that succeed return 1. This is useful e.g. to run "--stress"
tests on tests that fail 99% of the time on some setup, i.e. to smoke
out the flaky run which yielded success.

In a subsequent commit a new SANITIZE=leak mode will make use of this.

Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/t/t0000-basic.sh b/t/t0000-basic.sh
index 17a268ccd1b014201e25c810ebade41f6f31aecf..502b4bcf9ea0ad06cddaad50b999dd8024af28eb 100755 (executable)
--- a/t/t0000-basic.sh
+++ b/t/t0000-basic.sh
@@ -578,6 +578,78 @@ test_expect_success 'subtest: --run invalid range end' '
        EOF_ERR
 '
 
+test_expect_success 'subtest: --invert-exit-code without --immediate' '
+       run_sub_test_lib_test_err full-pass \
+               --invert-exit-code &&
+       check_sub_test_lib_test_err full-pass \
+               <<-\EOF_OUT 3<<-EOF_ERR
+       ok 1 - passing test #1
+       ok 2 - passing test #2
+       ok 3 - passing test #3
+       # passed all 3 test(s)
+       1..3
+       # faking up non-zero exit with --invert-exit-code
+       EOF_OUT
+       EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: all passed' '
+       run_sub_test_lib_test_err full-pass \
+               --invert-exit-code --immediate &&
+       check_sub_test_lib_test_err full-pass \
+               <<-\EOF_OUT 3<<-EOF_ERR
+       ok 1 - passing test #1
+       ok 2 - passing test #2
+       ok 3 - passing test #3
+       # passed all 3 test(s)
+       1..3
+       # faking up non-zero exit with --invert-exit-code
+       EOF_OUT
+       EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code without --immediate: partial pass' '
+       run_sub_test_lib_test partial-pass \
+               --invert-exit-code &&
+       check_sub_test_lib_test partial-pass <<-\EOF
+       ok 1 - passing test #1
+       not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+       #       false
+       ok 3 - passing test #3
+       # failed 1 among 3 test(s)
+       1..3
+       # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+       EOF
+'
+
+test_expect_success 'subtest: --invert-exit-code with --immediate: partial pass' '
+       run_sub_test_lib_test partial-pass \
+               --invert-exit-code --immediate &&
+       check_sub_test_lib_test partial-pass \
+               <<-\EOF_OUT 3<<-EOF_ERR
+       ok 1 - passing test #1
+       not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+       #       false
+       1..2
+       # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+       EOF_OUT
+       EOF_ERR
+'
+
+test_expect_success 'subtest: --invert-exit-code --immediate: got a failure' '
+       run_sub_test_lib_test partial-pass \
+               --invert-exit-code --immediate &&
+       check_sub_test_lib_test_err partial-pass \
+               <<-\EOF_OUT 3<<-EOF_ERR
+       ok 1 - passing test #1
+       not ok 2 - # TODO induced breakage (--invert-exit-code): failing test #2
+       #       false
+       1..2
+       # faked up failures as TODO & now exiting with 0 due to --invert-exit-code
+       EOF_OUT
+       EOF_ERR
+'
+
 test_expect_success 'subtest: tests respect prerequisites' '
        write_and_run_sub_test_lib_test prereqs <<-\EOF &&
 

diff --git a/t/test-lib.sh b/t/test-lib.sh
index 118720493bb02d7c01314d04afb1d2a366d1324c..31213b5f95b97f143899d0d7463b029fe6ee0577 100644 (file)
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -238,6 +238,9 @@ parse_option () {
                        ;;
                esac
                ;;
+       --invert-exit-code)
+               invert_exit_code=t
+               ;;
        *)
                echo "error: unknown test option '$opt'" >&2; exit 1 ;;
        esac
@@ -788,15 +791,31 @@ test_ok_ () {
        finalize_test_case_output ok "$@"
 }
 
+_invert_exit_code_failure_end_blurb () {
+       say_color warn "# faked up failures as TODO & now exiting with 0 due to --invert-exit-code"
+}
+
 test_failure_ () {
        failure_label=$1
        test_failure=$(($test_failure + 1))
-       say_color error "not ok $test_count - $1"
+       local pfx=""
+       if test -n "$invert_exit_code" # && test -n "$HARNESS_ACTIVE"
+       then
+               pfx="# TODO induced breakage (--invert-exit-code):"
+       fi
+       say_color error "not ok $test_count - ${pfx:+$pfx }$1"
        shift
        printf '%s\n' "$*" | sed -e 's/^/#      /'
        if test -n "$immediate"
        then
                say_color error "1..$test_count"
+               if test -n "$invert_exit_code"
+               then
+                       finalize_test_output
+                       _invert_exit_code_failure_end_blurb
+                       GIT_EXIT_OK=t
+                       exit 0
+               fi
                _error_exit
        fi
        finalize_test_case_output failure "$failure_label" "$@"
@@ -1229,7 +1248,14 @@ test_done () {
                        esac
                fi
 
-               if test -z "$debug" && test -n "$remove_trash"
+               if test -n "$stress" && test -n "$invert_exit_code"
+               then
+                       # We're about to move our "$TRASH_DIRECTORY"
+                       # to "$TRASH_DIRECTORY.stress-failed" if
+                       # --stress is combined with
+                       # --invert-exit-code.
+                       say "with --stress and --invert-exit-code we're not removing '$TRASH_DIRECTORY'"
+               elif test -z "$debug" && test -n "$remove_trash"
                then
                        test -d "$TRASH_DIRECTORY" ||
                        error "Tests passed but trash directory already removed before test cleanup; aborting"
@@ -1242,6 +1268,14 @@ test_done () {
                        } ||
                        error "Tests passed but test cleanup failed; aborting"
                fi
+
+               if test -z "$skip_all" && test -n "$invert_exit_code"
+               then
+                       say_color warn "# faking up non-zero exit with --invert-exit-code"
+                       GIT_EXIT_OK=t
+                       exit 1
+               fi
+
                test_at_end_hook_
 
                GIT_EXIT_OK=t
@@ -1254,6 +1288,13 @@ test_done () {
                        say "1..$test_count"
                fi
 
+               if test -n "$invert_exit_code"
+               then
+                       _invert_exit_code_failure_end_blurb
+                       GIT_EXIT_OK=t
+                       exit 0
+               fi
+
                GIT_EXIT_OK=t
                exit 1 ;;