Gather debug info on broken unit tests

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a15c13cb53b72986f395bb159cb61985db9a3367..193620da128d1e9172fc878dd2afa853e96f749e 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,6 +23,9 @@ variables:
 
   CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra 
 
+  # Pass run-time flags to AddressSanitizer to get core dumps on error.
+  ASAN_OPTIONS_COMMON: abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1
+
   TARBALL_COMPRESSOR: xz
   TARBALL_EXTENSION: xz
 
@@ -805,6 +808,8 @@ asan:sid:amd64:
   <<: *build_job
 
 system:asan:sid:amd64:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
   <<: *debian_sid_amd64_image
   <<: *system_test_job
   dependencies:
@@ -812,6 +817,8 @@ system:asan:sid:amd64:
   needs: ["asan:sid:amd64"]
 
 unit:asan:sid:amd64:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
   <<: *debian_sid_amd64_image
   <<: *unit_test_job
   dependencies:

diff --git a/unit/gdb b/unit/gdb
new file mode 100755 (executable)
index 0000000..68ccc10
--- /dev/null
+++ b/unit/gdb
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# `kyua debug` command does not work with libtool (see
+# https://github.com/jmmv/kyua/issues/207).  On some distributions `kyua debug`
+# runs the first `gdb` it finds in $PATH, but on Debian and Ubuntu it looks for
+# `/usr/bin/gdb`.  This script expects `gdb` to be moved to `gdb.orig` and
+# executed from there.
+coredump="$6"
+binary=$(gdb.orig --batch --core="${coredump}" 2>/dev/null | sed -ne "s/Core was generated by \`\(.*\)'./\1/p")
+# GDB 6.3 from OpenBSD 6.6 does not tell the full path of the broken binary.
+# We need to fix it. Either the binary or it's libtool script will do.
+if [ ! -e "${binary}" ]; then
+       binary="$(find "${TOP}" -name "${binary}" | head -n 1)"
+fi
+
+# $TOP points to BIND sources and should be set on `kyua debug` invocation.
+"${TOP}/libtool" --mode=execute gdb.orig \
+       --batch \
+       --command="${TOP}/bin/tests/system/run.gdb" \
+       --core="${coredump}" \
+       -- \
+       "${binary}"

diff --git a/unit/unittest.sh.in b/unit/unittest.sh.in
index ab087e0decbd46be50a59bafa96b84e6613b2f9a..b7b5ca06a4c60b500ddcb88a17d49c1f588d00db 100755 (executable)
--- a/unit/unittest.sh.in
+++ b/unit/unittest.sh.in
@@ -1,8 +1,16 @@
 #!/bin/sh
+
+# Find the top of the BIND9 tree.
+export TOP=@abs_top_builddir@
 KYUA=@KYUA@
 UNITTESTS=@UNITTESTS@
 CMOCKA_MESSAGE_OUTPUT=TAP
 export CMOCKA_MESSAGE_OUTPUT
+GDB="$(command -v gdb)"
+
+kyua_report() {
+       ${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
+}
 
 status=0
 if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
@@ -13,14 +21,50 @@ then
        ${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
        status=$?
 
-       ${KYUA} --logfile /dev/null report --results-file "${KYUA_RESULT:-LATEST}"
+       kyua_report
+
+       if command -v sysctl >/dev/null; then
+               if [ "$(uname -s)" = "Linux" ] && [ "$(sysctl -n kernel.core_uses_pid)" -ne 1 ]; then
+                       echo "kernel.core_uses_pid is not set on the Linux host"
+                       echo "kyua may not find core file of broken tests"
+               fi
+       else
+               echo "sysctl command is not present, can't check kernel.core_uses_pid."
+               echo "kyua may not find core file of broken tests"
+       fi
+
+       # Use kyua-debug(1) facility to gather additional data on failed tests.
+       # Some runs will just show verbose information from the run, some will
+       # show backtrace via gdb(1).
+       broken_tests="$(kyua_report | awk '/Broken tests/ { flag=1; next } /Summary/ { flag=0 } flag' | awk '{ print $1 }')"
+       if [ -n "${CI}" ] && [ "$(id -u)" -eq 0 ] && [ -n "${broken_tests}" ] && [ -n "${GDB}" ]; then
+               if grep '^#define USE_LIBTOOL 1$' "${TOP}/config.h" >/dev/null; then
+                       # kyua debug command misidentifies broken binary when libtool is used
+                       # to configure BIND (see https://github.com/jmmv/kyua/issues/207).
+                       # Here we try "trick" kyua use our custom gdb script instead
+                       # of using gdb(1) directly. That's why this part needs to be run as root
+                       # and, for safety reasons, only in the CI.
+                       mv "${GDB}" "${GDB}.orig"
+                       cp "${TOP}/unit/gdb" "${GDB}"
+                       for test in ${broken_tests}; do
+                               echo
+                               ${KYUA} debug "${test}"
+                       done
+                       mv "${GDB}.orig" "${GDB}"
+               else
+                       for test in ${broken_tests}; do
+                               echo
+                               ${KYUA} debug "${test}"
+                       done
+               fi
+       fi
 
-       if [ "${status}" -eq "0" ]
+       if [ "${status}" -eq 0 ]
        then
                rm -f kyua.log
                echo "R:PASS"
        else
-               echo "R:FAIL"
+               echo "R:FAIL:status:${status}"
        fi
        echo "E:unit:$(date)"
 fi

diff --git a/util/copyrights b/util/copyrights
index 4c2a89a862670268da56dfbd091b78e695974af1..d891c5ce276d0964434f8b07d9b7f52f92c75eb7 100644 (file)
--- a/util/copyrights
+++ b/util/copyrights
@@ -2560,6 +2560,7 @@
 ./make/rules.in                                        MAKE    1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020
 ./mkinstalldirs                                        X       1996,2018,2019,2020
 ./unit/README                                  X       2011,2013,2018,2019,2020
+./unit/gdb                                     X       2020
 ./unit/unittest.sh.in                          X       2011,2012,2015,2018,2019,2020
 ./util/COPYRIGHT                               X       1996,1997,1998,1999,2000,2001,2004,2007,2016,2018,2019,2020
 ./util/COPYRIGHT.BRIEF                         X       1996,1997,1998,1999,2000,2001,2004,2016,2018,2019,2020