From 0fea6a7f8e7ecdab81c892f044baec577f4d0bb5 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Mon, 28 Nov 2022 17:01:45 +0100 Subject: [PATCH] github: Adapt to switch to Ubuntu 22.04 for ubuntu-latest Ubuntu 22.04 ships OpenSSL 3, which requires debug symbols so we can whitelist leaks because we don't deinitialize the library. And because the shipped library is not built with `-fno-omit-frame-pointer`, the build with AddressSanitizer can't use its fast stack unwind method. However, the previous workaround for DTLS handling with glibc apparently isn't necessary anymore. In the custom OpenSSL build we drop no-stdio as that lets the configure check for libldns fail because ERR_print_errors_fp@OPENSSL_3.0.0 is not found. For ccache, the default path to the cache directory has changed. Also simplified the NM tests as there is only one build since 085daf474330 ("nm: Remove old libnm-glib compat stuff"). --- .github/workflows/android.yml | 2 +- .github/workflows/linux.yml | 57 +++++++++++++++++++------------- .github/workflows/sonarcloud.yml | 2 +- .github/workflows/windows.yml | 2 +- .lsan.suppressions | 1 + scripts/test.sh | 52 +++++++++++++++++++++-------- 6 files changed, 77 insertions(+), 39 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 954f810715..37691286af 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/cache@v2 with: - path: ~/.ccache + path: ~/.cache/ccache key: ccache-android-${{ github.sha }} restore-keys: | ccache-android- diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cd1dc7bb60..92ba032074 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -44,7 +44,7 @@ jobs: - test: apidoc - test: coverage - test: dist - - test: nm-no-glib + - test: nm - test: fuzzing compiler: clang monolithic: yes @@ -53,21 +53,19 @@ jobs: MONOLITHIC: ${{ matrix.monolithic || 'no' }} CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} - # LSan causes spurious SIGSEGV after tests due to DTLS handling by glibc - ASAN_OPTIONS: intercept_tls_get_addr=0 steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 with: - path: ~/.ccache + path: ~/.cache/ccache # with regards to ccache, monolithic builds don't differ from regular # builds and, similarly, builds with leak-detective only differ in two # files (LD itself and library.c); but different tests build different # dependencies, so different caches are needed - key: ccache-${{ runner.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + key: ccache-ubuntu-latest-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} restore-keys: | - ccache-${{ runner.os }}-${{ env.CC }}-${{ matrix.test }}- - ccache-${{ runner.os }}-${{ env.CC }}- + ccache-ubuntu-latest-${{ env.CC }}-${{ matrix.test }}- + ccache-ubuntu-latest-${{ env.CC }}- - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV @@ -86,26 +84,38 @@ jobs: crypto-plugins: needs: pre-check if: ${{ needs.pre-check.outputs.should_skip != 'true' }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ ubuntu-latest, ubuntu-20.04 ] test: [ botan, wolfssl, openssl, openssl-3, gcrypt ] leak-detective: [ no, yes ] + exclude: + # test custom-built libs only on one platform + - os: ubuntu-20.04 + test: botan + - os: ubuntu-20.04 + test: wolfssl + - os: ubuntu-20.04 + test: openssl-3 env: LEAK_DETECTIVE: ${{ matrix.leak-detective || 'no' }} + CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 with: - path: ~/.ccache - key: ccache-${{ runner.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + # path is different on newer systems + path: | + ~/.cache/ccache + ~/.ccache + key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} restore-keys: | - ccache-${{ runner.os }}-${{ env.CC }}-${{ matrix.test }}- - ccache-${{ runner.os }}-${{ env.CC }}- - ccache-${{ runner.os }}-${{ env.CC }}-all-${{ github.sha }} - ccache-${{ runner.os }}-${{ env.CC }}-all- - ccache-${{ runner.os }}-${{ env.CC }}- + ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}- + ccache-${{ matrix.os }}-${{ env.CC }}-all-${{ github.sha }} + ccache-${{ matrix.os }}-${{ env.CC }}-all- + ccache-${{ matrix.os }}-${{ env.CC }}- - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV @@ -119,30 +129,31 @@ jobs: path: config.log retention-days: 5 - bionic: + older: needs: pre-check if: ${{ needs.pre-check.outputs.should_skip != 'true' }} - runs-on: ubuntu-18.04 + runs-on: ${{ matrix.os }} strategy: matrix: - test: [ all ] + os: [ ubuntu-20.04, ubuntu-18.04 ] + test: [ all, nm ] compiler: [ gcc, clang ] - include: + exclude: - test: nm + compiler: clang env: LEAK_DETECTIVE: ${{ matrix.leak-detective || 'no' }} CC: ${{ matrix.compiler || 'gcc' }} TEST: ${{ matrix.test }} - UBUNTU_BIONIC: yes steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-bionic-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} + key: ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}-${{ github.sha }} restore-keys: | - ccache-bionic-${{ env.CC }}-${{ matrix.test }}- - ccache-bionic-${{ env.CC }}- + ccache-${{ matrix.os }}-${{ env.CC }}-${{ matrix.test }}- + ccache-${{ matrix.os }}-${{ env.CC }}- - run: | sudo apt-get install -qq ccache echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index c647579047..62b15a829c 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/cache@v2 with: path: | - ~/.ccache + ~/.cache/ccache ~/.sonar-cache key: ccache-sonarcloud-${{ github.sha }} restore-keys: | diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3e0c29ab36..7dd410691b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/cache@v2 with: - path: ~/.ccache + path: ~/.cache/ccache key: ccache-${{ runner.os }}-${{ matrix.test }}-${{ github.sha }} restore-keys: | ccache-${{ runner.os }}-${{ matrix.test }}- diff --git a/.lsan.suppressions b/.lsan.suppressions index be166e53b1..5978d69d8d 100644 --- a/.lsan.suppressions +++ b/.lsan.suppressions @@ -5,3 +5,4 @@ leak:EVP_RAND_fetch leak:OSSL_DECODER_do_all_provided leak:OSSL_ENCODER_do_all_provided leak:OSSL_PROVIDER_load +leak:OSSL_PROVIDER_try_load diff --git a/scripts/test.sh b/scripts/test.sh index b03454db53..d528d4bbea 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -97,7 +97,7 @@ build_openssl() SSL_SRC=https://www.openssl.org/source/$SSL_PKG.tar.gz SSL_INS=$DEPS_PREFIX/ssl SSL_OPT="-d shared no-tls no-dtls no-ssl3 no-zlib no-comp no-idea no-psk no-srp - no-stdio no-tests enable-rfc3779 enable-ec_nistp_64_gcc_128" + no-tests enable-rfc3779 enable-ec_nistp_64_gcc_128" if test -d "$SSL_DIR"; then return @@ -129,6 +129,35 @@ use_custom_openssl() fi } +system_uses_openssl3() +{ + pkg-config --atleast-version=3.0.0 libcrypto + return $? +} + +prepare_system_openssl() +{ + # On systems that ship OpenSSL 3 (e.g. Ubuntu 22.04), we require debug + # symbols to whitelist leaks + if test "$1" = "deps"; then + echo "deb http://ddebs.ubuntu.com $(lsb_release -cs) main restricted + deb http://ddebs.ubuntu.com $(lsb_release -cs)-updates main restricted + deb http://ddebs.ubuntu.com $(lsb_release -cs)-proposed main restricted" | \ + sudo tee -a /etc/apt/sources.list.d/ddebs.list + sudo apt-get install -qq ubuntu-dbgsym-keyring + DEPS="$DEPS libssl3-dbgsym" + fi + if test "$LEAK_DETECTIVE" = "yes"; then + # make sure we can properly whitelist functions with leak detective + DEPS="$DEPS binutils-dev" + CONFIG="$CONFIG --enable-bfd-backtraces" + else + # with ASan we have to use the (extremely) slow stack unwind as the + # shipped version of the library is built with -fomit-frame-pointer + export ASAN_OPTIONS=fast_unwind_on_malloc=0 + fi +} + : ${BUILD_DIR=$PWD} : ${DEPS_BUILD_DIR=$BUILD_DIR/..} : ${DEPS_PREFIX=/usr/local} @@ -157,15 +186,17 @@ openssl*) if test "$TEST" = "openssl-3"; then DEPS="" use_custom_openssl $1 + elif system_uses_openssl3; then + prepare_system_openssl $1 fi ;; gcrypt) CONFIG="--disable-defaults --enable-pki --enable-gcrypt --enable-random --enable-pem --enable-pkcs1 --enable-pkcs8 --enable-gcm --enable-hmac --enable-kdf -enable-curve25519 --enable-x509 --enable-constraints" export TESTS_PLUGINS="test-vectors gcrypt! random pem pkcs1 pkcs8 gcm hmac kdf curve25519 x509 constraints" - if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "20.04" ]; then - DEPS="libgcrypt20-dev" - else + if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "18.04" ]; then DEPS="libgcrypt11-dev" + else + DEPS="libgcrypt20-dev" fi ;; botan) @@ -217,10 +248,10 @@ all|coverage|sonarcloud) libldap2-dev libpcsclite-dev libpam0g-dev binutils-dev libnm-dev libgcrypt20-dev libjson-c-dev python3-pip libtspi-dev libsystemd-dev libselinux1-dev" - if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "20.04" ]; then - DEPS="$DEPS libiptc-dev" - else + if [ "$ID" = "ubuntu" -a "$VERSION_ID" = "18.04" ]; then DEPS="$DEPS iptables-dev python3-setuptools" + else + DEPS="$DEPS libiptc-dev" fi PYDEPS="tox" if test "$1" = "build-deps"; then @@ -348,13 +379,8 @@ fuzzing) symbolize=1:handle_segv=1:fast_unwind_on_fatal=0:external_symbolizer_path=/usr/bin/llvm-symbolizer-3.5 fi ;; -nm|nm-no-glib) +nm) DEPS="gnome-common libsecret-1-dev libgtk-3-dev libnm-dev libnma-dev" - if test "$TEST" = "nm"; then - DEPS="$DEPS libnm-glib-vpn-dev libnm-gtk-dev" - else - CONFIG="$CONFIG --without-libnm-glib" - fi cd src/frontends/gnome # don't run ./configure with ./autogen.sh export NOCONFIGURE=1 -- 2.47.2