test/run-integration-tests.sh

   1 #!/usr/bin/env bash
   2 # SPDX-License-Identifier: LGPL-2.1-or-later
   3 set -e
   4
   5 is_valid_target() {
   6     local target="${1:?}"
   7     local t
   8
   9     for t in all setup run clean clean-again; do
  10         [[ "$target" == "$t" ]] && return 0
  11     done
  12
  13     return 1
  14 }
  15
  16 pass_deny_list() {
  17     local test="${1:?}"
  18     local marker
  19
  20     for marker in $DENY_LIST_MARKERS $BLACKLIST_MARKERS; do
  21         if [[ -f "$test/$marker" ]]; then
  22             echo "========== DENY-LISTED: $test ($marker) =========="
  23             return 1
  24         fi
  25     done
  26
  27     return 0
  28 }
  29
  30 test_run() {
  31     local test_name="${1:?}"
  32     shift
  33
  34     if [[ $# -eq 0 ]]; then
  35         echo >&2 "test_run: missing arguments"
  36         exit 1
  37     fi
  38
  39     # Note: let's be very explicit in reporting the return code of the test command here, i.e don't rely on
  40     #       `set -e` or the return code of the last statement in the function, since reporting false positive
  41     #       would be very bad in this case.
  42     if [[ "${SPLIT_TEST_LOGS:-0}" -ne 0 && -n "${ARTIFACT_DIRECTORY:-}" ]]; then
  43         (set -x; "$@") &>>"$ARTIFACT_DIRECTORY/$test_name.log" || return $?
  44     else
  45         (set -x; "$@") || return $?
  46     fi
  47 }
  48
  49 ARGS=(setup run clean-again)
  50 CLEAN=0
  51 CLEAN_AGAIN=0
  52 COUNT=0
  53 FAILURES=0
  54 declare -A RESULTS
  55 declare -A TIMES
  56
  57 if [[ "${NO_BUILD:-0}" =~ ^(1|yes|true)$ ]]; then
  58     BUILD_DIR=""
  59 elif BUILD_DIR="$("$(dirname "$0")/../tools/find-build-dir.sh")"; then
  60     ninja -C "$BUILD_DIR"
  61 else
  62     echo >&2 "No build found, please set BUILD_DIR or NO_BUILD"
  63     exit 1
  64 fi
  65
  66 if [[ $# -gt 0 ]]; then
  67     ARGS=("$@")
  68 fi
  69
  70 # Reject invalid make targets
  71 for arg in "${ARGS[@]}"; do
  72     if ! is_valid_target "$arg"; then
  73         echo >&2 "Invalid target: $arg"
  74         exit 1
  75     fi
  76 done
  77
  78 # Separate 'clean' and 'clean-again' operations
  79 args_filtered=()
  80 for arg in "${ARGS[@]}"; do
  81     if [[ "$arg" == "clean-again" ]]; then
  82         CLEAN_AGAIN=1
  83     elif [[ "$arg" == "clean" ]]; then
  84         CLEAN=1
  85     else
  86         args_filtered+=("$arg")
  87     fi
  88 done
  89 ARGS=("${args_filtered[@]}")
  90
  91 cd "$(dirname "$0")"
  92
  93 SELECTED_TESTS="${SELECTED_TESTS:-TEST-??-*}"
  94
  95 # Let's always do the cleaning operation first, because it destroys the image
  96 # cache.
  97 if [[ $CLEAN -eq 1 ]]; then
  98     for test in $SELECTED_TESTS; do
  99         test_run "$test" make -C "$test" clean
 100     done
 101 fi
 102
 103 # Run actual tests (if requested)
 104 if [[ ${#ARGS[@]} -ne 0 ]]; then
 105     for test in $SELECTED_TESTS; do
 106         COUNT=$((COUNT + 1))
 107
 108         pass_deny_list "$test" || continue
 109         SECONDS=0
 110
 111         echo -e "\n[$(date +%R:%S)] --x-- Running $test --x--"
 112         set +e
 113         test_run "$test" make -C "$test" "${ARGS[@]}"
 114         result=$?
 115         set -e
 116         echo "[$(date +%R:%S)] --x-- Result of $test: $result --x--"
 117
 118         RESULTS["$test"]="$result"
 119         TIMES["$test"]="$SECONDS"
 120
 121         # Run clean-again here to free up space, if requested, and if the test succeeded
 122         if [[ "$result" -ne 0 ]]; then
 123             FAILURES=$((FAILURES + 1))
 124         elif [[ $CLEAN_AGAIN -eq 1 ]]; then
 125             test_run "$test" make -C "$test" clean-again
 126         fi
 127     done
 128 fi
 129
 130 echo ""
 131
 132 for test in "${!RESULTS[@]}"; do
 133     result="${RESULTS[$test]}"
 134     time="${TIMES[$test]}"
 135     [[ "$result" -eq 0 ]] && string="SUCCESS" || string="FAIL"
 136     printf "%-35s %-8s (%3s s)\n" "$test:" "$string" "$time"
 137 done | sort
 138
 139 if [[ "$FAILURES" -eq 0 ]]; then
 140     echo -e "\nALL $COUNT TESTS PASSED"
 141 else
 142     echo -e "\nTOTAL FAILURES: $FAILURES OF $COUNT"
 143 fi
 144
 145 # If we have coverage files, merge them into a single report for upload
 146 if [[ -n "$ARTIFACT_DIRECTORY" ]]; then
 147     lcov_args=()
 148
 149     while read -r info_file; do
 150         lcov_args+=(--add-tracefile "$info_file")
 151     done < <(find "$ARTIFACT_DIRECTORY" -maxdepth 1 -name "*.coverage-info")
 152
 153     if [[ ${#lcov_args[@]} -gt 1 ]]; then
 154         lcov "${lcov_args[@]}" --output-file "$ARTIFACT_DIRECTORY/merged.coverage-info"
 155     fi
 156 fi
 157
 158 exit "$FAILURES"