Enhance unit test debugging

When unit test fails, core file is created. Kyua's 'debug' command can
run GDB on it and provide backtrace. Unfortunately Kyua is picky about
location of these core files we opt to use custom Kyua fork and copy
core files from Kyua working directory to source tree and make it
available in GitLab.

(cherry picked from commit 8fad74e0e5c7cf9338a739ed1e937619cdb4d55a)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5685a9d800c89fd44cd28dc6d60bf960b3e5f9d4..2ec5ee50da4e9d9494b0acbfd2373149b98b4054 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -318,10 +318,7 @@ stages:
   after_script:
     - *kyua_report_html
   artifacts:
-    paths:
-      - kyua.log
-      - kyua.results
-      - kyua_html/
+    untracked: true
     expire_in: "1 day"
     when: on_failure
 
@@ -1099,8 +1096,6 @@ system:clang:freebsd11.3:amd64:
 unit:clang:freebsd11.3:amd64:
   <<: *freebsd_amd64
   <<: *unit_test_job
-  variables:
-    USER: gitlab-runner
   dependencies:
     - clang:freebsd11.3:amd64
   needs: ["clang:freebsd11.3:amd64"]
@@ -1128,8 +1123,6 @@ system:clang:freebsd12.1:amd64:
 unit:clang:freebsd12.1:amd64:
   <<: *freebsd_amd64
   <<: *unit_test_job
-  variables:
-    USER: gitlab-runner
   dependencies:
     - clang:freebsd12.1:amd64
   needs: ["clang:freebsd12.1:amd64"]

diff --git a/unit/unittest.sh.in b/unit/unittest.sh.in
index 60b91274bebaf15550ee8b722c1439583112f5ea..656b4612313741ceccbee443fcc8e113b7775871 100755 (executable)
--- a/unit/unittest.sh.in
+++ b/unit/unittest.sh.in
@@ -12,50 +12,64 @@ kyua_report() {
        ${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
 }
 
+clear_kyua_work_dir() {
+       KYUA_WORK_DIR="$(grep -i -m1 "failed" "${1}" | sed -n 's|.*\(/tmp/kyua\.[A-Za-z0-9]*\).*|\1|p')"
+       if [ -n "${CI}" ] && [ -d "${KYUA_WORK_DIR}" ]; then
+               find "${KYUA_WORK_DIR}" \( -name 'core*' -o -name '*.core' \) -exec mv -v {} . \;
+               rm -rf "${KYUA_WORK_DIR}"
+       fi
+}
+
 status=0
 if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
 then
        echo "S:unit:$(date)"
        echo "T:unit:1:A"
-       echo "I: unit tests (using kyua)"
+       echo "I:unit tests (using kyua)"
+
        ${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
        status=$?
 
        kyua_report
 
-       if command -v sysctl >/dev/null; then
-               if [ "$(uname -s)" = "Linux" ] && [ "$(sysctl -n kernel.core_uses_pid)" -ne 1 ]; then
-                       echo "kernel.core_uses_pid is not set on the Linux host"
-                       echo "kyua may not find core file of broken tests"
-               fi
-       else
-               echo "sysctl command is not present, can't check kernel.core_uses_pid."
-               echo "kyua may not find core file of broken tests"
-       fi
+       clear_kyua_work_dir kyua.log
 
        # Use kyua-debug(1) facility to gather additional data on failed tests.
        # Some runs will just show verbose information from the run, some will
        # show backtrace via gdb(1).
-       broken_tests=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
-       if [ -n "${CI}" ] && [ "$(id -u)" -eq 0 ] && [ -n "${broken_tests}" ] && [ -n "${GDB}" ]; then
+       USER_ID=$(id -u)
+       BROKEN_TESTS=$(kyua_report | awk '$2 == "->" && ( $3 == "broken:" || $3 == "failed:" ) { print $1 }')
+       # Conditions for getting kyua debug info and GDB backtrace: runs under CI
+       # (safety), GDB present, root privileges, failed tests.
+       if [ -n "${CI}" ] && [ -n "${GDB}" ] && [ "${USER_ID:-1}" -eq 0 ] && [ -n "${BROKEN_TESTS}" ]; then
+               if [ "$(uname -s)" = "Linux" ] && ! sysctl -n "kernel.core_pattern" | grep -xq "core.%p"; then
+                       echo "I:*** kernel.core_pattern is not set to 'core.%p'"
+                       echo "I:*** kyua may not be able to find core dumps for broken tests"
+               fi
+               if [ "$(uname -s)" = "FreeBSD" ] && ! sysctl -n "kern.corefile" | grep -xq "core.%P"; then
+                       echo "I:*** kern.corefile is not set to 'core.%P'"
+                       echo "I:*** kyua may not be able to find core dumps for broken tests"
+               fi
                if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
-                       # kyua debug command misidentifies broken binary when libtool is used
-                       # to configure BIND (see https://github.com/jmmv/kyua/issues/207).
-                       # Here we try "trick" kyua use our custom gdb script instead
-                       # of using gdb(1) directly. That's why this part needs to be run as root
+                       # kyua debug command misidentifies broken binaries when libtool
+                       # is used (see https://github.com/jmmv/kyua/issues/207).
+                       # Here we try to "trick" kyua to use our custom gdb script instead
+                       # of using gdb(1) directly. Hence this part needs to be run as root
                        # and, for safety reasons, only in the CI.
                        mv "${GDB}" "${GDB}.orig"
                        cp "${TOP}/unit/gdb" "${GDB}"
-                       for test in ${broken_tests}; do
-                               echo
-                               ${KYUA} debug "${test}"
-                       done
+               fi
+               i=1
+               for test in ${BROKEN_TESTS}; do
+                       echo
+                       echo "----- $test -----"
+                       KYUA_DEBUG_LOG="kyua.debug.log.${i}"
+                       ${KYUA} debug "${test}" 2>&1 | tee "${KYUA_DEBUG_LOG}"
+                       clear_kyua_work_dir "${KYUA_DEBUG_LOG}"
+                       i=$((i + 1))
+               done
+               if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
                        mv "${GDB}.orig" "${GDB}"
-               else
-                       for test in ${broken_tests}; do
-                               echo
-                               ${KYUA} debug "${test}"
-                       done
                fi
        fi