github: Enable AddressSanitizer if leak-detective is disabled

At least for the tests where it is available and works.  It conflicts
with the instrumentation used by the coverage and fuzzing (and possibly
sonarcloud) tests, the toolchain for the Windows builds doesn't seem to
support it, and on FreeBSD the test executables hang due to a
compatibility issue with FreeBSD's qsort(), which has been fixed [1],
but that has not made it into the clang version in the base system.

For the custom OpenSSL build, debug symbols are enabled so we can
suppress some leaks properly.

[1] https://github.com/llvm/llvm-project/issues/46176

diff --git a/scripts/test.sh b/scripts/test.sh
index 63d36f218990ca50fc5ff8ea25d3ce91445606fc..9e102553ad99c285fe6090f784944e84cc3cc99a 100755 (executable)
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -96,7 +96,7 @@ build_openssl()
        SSL_DIR=$DEPS_BUILD_DIR/$SSL_PKG
        SSL_SRC=https://www.openssl.org/source/$SSL_PKG.tar.gz
        SSL_INS=$DEPS_PREFIX/ssl
-       SSL_OPT="shared no-tls no-dtls no-ssl3 no-zlib no-comp no-idea no-psk no-srp
+       SSL_OPT="-d shared no-tls no-dtls no-ssl3 no-zlib no-comp no-idea no-psk no-srp
                         no-stdio no-tests enable-rfc3779 enable-ec_nistp_64_gcc_128"
 
        if test -d "$SSL_DIR"; then
@@ -471,6 +471,21 @@ CONFIG="$CONFIG
        --enable-monolithic=${MONOLITHIC-no}
        --enable-leak-detective=${LEAK_DETECTIVE-no}"
 
+case "$TEST" in
+       coverage|freebsd|fuzzing|sonarcloud|win*)
+               # don't use AddressSanitizer if it's not available or causes conflicts
+               CONFIG="$CONFIG --disable-asan"
+               ;;
+       *)
+               if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "18.04" ]; then
+                       # the libstdc++ workaround for libbotan doesn't work on Ubuntu 18.04
+                       CONFIG="$CONFIG --disable-asan"
+               elif [ "$LEAK_DETECTIVE" != "yes" ]; then
+                       CONFIG="$CONFIG --enable-asan"
+               fi
+               ;;
+esac
+
 echo "$ ./autogen.sh"
 ./autogen.sh || exit $?
 echo "$ CC=$CC CFLAGS=\"$CFLAGS\" ./configure $CONFIG"