From: Ondřej Surý Date: Tue, 24 Mar 2020 08:43:45 +0000 (+0100) Subject: Adjust the GitLab CI jobs to match the new images X-Git-Tag: v9.14.12~10^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a58f256c223fe22128cfe1819dac04e864f25069;p=thirdparty%2Fbind9.git Adjust the GitLab CI jobs to match the new images The custom builds (oot, asan, tsan) were mostly built using Debian sid amd64 image. The problem was that this image broke too easily, because it's Debian "unstable" after all. This commit introduces "base_image" that should be most stable with extra bits on top (clang, coccinelle, cppcheck, ...). Currently, that would be Debian buster amd64. Other changes introduced by this commit: * Change the default clang version to 10 * Run both ASAN and TSAN with both gcc and clang compilers * Remove Clang Debian stretch i386 job (cherry picked from commit 5f5721aa11a88cfb9d2bb65fa8b8044fcb0022ed) (cherry picked from commit 7398668e76c27d21db37e67c1617a7cfaffd4dbd) --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fada2be1d7a..355fe00b25e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,11 @@ variables: MAKE_COMMAND: make CONFIGURE: ./configure - SCAN_BUILD: scan-build-9 + CLANG: clang-10 + SCAN_BUILD: scan-build-10 + SYMBOLIZER: /usr/lib/llvm-10/bin/llvm-symbolizer + ASAN_SYMBOLIZER_PATH: "$SYMBOLIZER" + CLANG_FORMAT: clang-format-10 CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra @@ -135,6 +139,12 @@ stages: image: "$CI_REGISTRY_IMAGE:ubuntu-bionic-i386" <<: *linux_i386 +# Base image +# This is a meta image that is used as a base for non-specific jobs + +.base: &base_image + <<: *debian_buster_amd64_image + ### Job Templates .default-triggering-rules: &default_triggering_rules @@ -155,12 +165,12 @@ stages: .precheck: &precheck_job <<: *default_triggering_rules - <<: *debian_sid_amd64_image + <<: *base_image stage: precheck .autoconf: &autoconf_job <<: *release_branch_triggering_rules - <<: *debian_sid_amd64_image + <<: *base_image stage: precheck script: - autoreconf -fi @@ -195,7 +205,7 @@ stages: - test -z "${RUN_MAKE_INSTALL}" || make install - test -z "${RUN_MAKE_INSTALL}" || sh util/check-make-install needs: - - job: autoreconf:sid:amd64 + - job: autoreconf artifacts: true artifacts: untracked: true @@ -224,7 +234,7 @@ stages: - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"' - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln' needs: - - job: autoreconf:sid:amd64 + - job: autoreconf artifacts: false artifacts: untracked: true @@ -344,10 +354,10 @@ stages: # Jobs in the precheck stage -autoreconf:sid:amd64: +autoreconf: <<: *autoconf_job -misc:sid:amd64: +misc: <<: *precheck_job script: - sh util/check-ans-prereq.sh @@ -375,17 +385,23 @@ misc:sid:amd64: expire_in: "1 day" when: on_failure -🐞:sid:amd64: +clang-format: + <<: *precheck_job + needs: [] + script: + - if [ -r .clang-format ]; then "${CLANG_FORMAT}" -i -style=file $(git ls-files '*.c' '*.h'); fi + - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi + +coccinelle: <<: *precheck_job - <<: *debian_buster_amd64_image needs: [] script: - util/check-cocci - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi -tarball-create:sid:amd64: +tarball-create: stage: precheck - <<: *debian_sid_amd64_image + <<: *base_image script: - source version - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}" @@ -402,27 +418,27 @@ tarball-create:sid:amd64: # Jobs for doc builds on Debian Sid (amd64) -docs:sid:amd64: +docs: <<: *release_branch_triggering_rules - <<: *debian_sid_amd64_image + <<: *base_image stage: docs script: - ./configure || cat config.log - make -C doc/misc docbook - make -C doc/arm Bv9ARM.html needs: - - job: autoreconf:sid:amd64 + - job: autoreconf artifacts: true artifacts: paths: - doc/arm/ expire_in: "1 month" -push:docs:sid:amd64: - <<: *debian_sid_amd64_image +push:docs: + <<: *base_image stage: push needs: - - job: docs:sid:amd64 + - job: docs artifacts: false script: - curl -X POST -F token=$GITLAB_PAGES_DOCS_TRIGGER_TOKEN -F ref=master $GITLAB_PAGES_DOCS_TRIGGER_URL @@ -582,9 +598,9 @@ unit:gcc:buster:amd64: -o scan-build.reports \ make -j${BUILD_PARALLEL_JOBS:-1} all V=1 -scan-build:buster:amd64: +scan-build: <<: *default_triggering_rules - <<: *debian_buster_amd64_image + <<: *base_image stage: postcheck variables: CC: clang-9 @@ -595,7 +611,7 @@ scan-build:buster:amd64: - *configure - *scan_build needs: - - job: autoreconf:sid:amd64 + - job: autoreconf artifacts: true artifacts: paths: @@ -629,8 +645,8 @@ unit:gcc:sid:amd64: - job: gcc:sid:amd64 artifacts: true -cppcheck:gcc:sid:amd64: - <<: *debian_sid_amd64_image +cppcheck: + <<: *base_image <<: *cppcheck_job needs: - job: gcc:sid:amd64 @@ -638,66 +654,90 @@ cppcheck:gcc:sid:amd64: # Job for out-of-tree GCC build on Debian Sid (amd64) -oot:sid:amd64: +out-of-tree: variables: CC: gcc - CFLAGS: "${CFLAGS_COMMON} -O3" + CFLAGS: "${CFLAGS_COMMON} -Og" CONFIGURE: ../configure EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2" RUN_MAKE_INSTALL: 1 OOT_BUILD_WORKSPACE: workspace - <<: *debian_sid_amd64_image + <<: *base_image <<: *build_job # Jobs for tarball GCC builds on Debian Sid (amd64) -tarball:sid:amd64: +tarball: variables: CC: gcc EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2" RUN_MAKE_INSTALL: 1 - <<: *debian_sid_amd64_image + <<: *base_image <<: *build_job before_script: - tar --extract --file bind-*.tar.${TARBALL_EXTENSION} - rm -f bind-*.tar.${TARBALL_EXTENSION} - cd bind-* needs: - - job: tarball-create:sid:amd64 + - job: tarball-create artifacts: true only: - tags -system:tarball:sid:amd64: - <<: *debian_sid_amd64_image +system:tarball: + <<: *base_image <<: *system_test_job before_script: - cd bind-* - *setup_interfaces needs: - - job: tarball:sid:amd64 + - job: tarball artifacts: true only: - tags -unit:tarball:sid:amd64: - <<: *debian_sid_amd64_image +unit:tarball: + <<: *base_image <<: *unit_test_job before_script: - cd bind-* needs: - - job: tarball:sid:amd64 + - job: tarball artifacts: true only: - tags +# Jobs for regular GCC builds on Debian Sid (arm64) + +gcc:sid:arm64: + variables: + CC: gcc + CFLAGS: "${CFLAGS_COMMON} -Og" + EXTRA_CONFIGURE: "--with-libidn2" + <<: *debian_sid_arm64_image + <<: *build_job + +system:gcc:sid:arm64: + <<: *debian_sid_arm64_image + <<: *system_test_job + needs: + - job: gcc:sid:arm64 + artifacts: true + +unit:gcc:sid:arm64: + <<: *debian_sid_arm64_image + <<: *unit_test_job + needs: + - job: gcc:sid:arm64 + artifacts: true + # Jobs for regular GCC builds on Debian Sid (i386) gcc:sid:i386: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2 --without-python" + EXTRA_CONFIGURE: "--with-libidn2 --without-python" <<: *debian_sid_i386_image <<: *build_job @@ -811,40 +851,202 @@ unit:gcc:bionic:amd64: - job: gcc:bionic:amd64 artifacts: true -# Jobs for GCC builds with ASAN enabled on Debian Sid (amd64) +# Jobs for builds with ASAN enabled -asan:sid:amd64: +gcc:asan: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined -DISC_MEM_USE_INTERNAL_MALLOC=0" LDFLAGS: "-fsanitize=address,undefined" EXTRA_CONFIGURE: "--with-libidn2" - <<: *debian_sid_amd64_image + <<: *base_image <<: *build_job -system:asan:sid:amd64: +system:gcc:asan: variables: ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON} - <<: *debian_sid_amd64_image + <<: *base_image <<: *system_test_job needs: - - job: asan:sid:amd64 + - job: gcc:asan artifacts: true -unit:asan:sid:amd64: +unit:gcc:asan: variables: ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON} - <<: *debian_sid_amd64_image + <<: *base_image <<: *unit_test_job needs: - - job: asan:sid:amd64 + - job: gcc:asan + artifacts: true + +clang:asan: + variables: + CC: ${CLANG} + CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined -DISC_MEM_USE_INTERNAL_MALLOC=0" + LDFLAGS: "-fsanitize=address,undefined" + EXTRA_CONFIGURE: "--with-libidn2" + <<: *base_image + <<: *build_job + +system:clang:asan: + variables: + ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON} + <<: *base_image + <<: *system_test_job + needs: + - job: gcc:asan + artifacts: true + +unit:clang:asan: + variables: + ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON} + <<: *base_image + <<: *unit_test_job + needs: + - job: gcc:asan + artifacts: true + +# Jobs for builds with TSAN enabled + +gcc:tsan: + <<: *base_image + <<: *build_job + variables: + CC: gcc + CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0" + LDFLAGS: "-fsanitize=thread" + EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock" + +system:gcc:tsan: + variables: + TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0" + before_script: + - *setup_interfaces + - echo $TSAN_OPTIONS + <<: *base_image + <<: *system_test_job + needs: + - job: gcc:tsan + artifacts: true + allow_failure: true + after_script: + - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; + artifacts: + expire_in: "1 day" + paths: + - bin/tests/system/*/tsan.* + - bin/tests/system/*/*/tsan.* + - tsan/ + when: on_failure + +unit:gcc:tsan: + variables: + TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER" + before_script: + - echo $TSAN_OPTIONS + - lib/isc/tests/result_test + <<: *base_image + <<: *unit_test_job + needs: + - job: gcc:tsan + artifacts: true + allow_failure: true + after_script: + - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; + artifacts: + expire_in: "1 day" + paths: + - lib/*/tests/tsan.* + - tsan/ + - kyua.log + - kyua.results + - kyua_html/ + when: on_failure + +clang:tsan: + <<: *base_image + <<: *build_job + variables: + CC: "${CLANG}" + CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0" + LDFLAGS: "-fsanitize=thread" + EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock" + +system:clang:tsan: + variables: + TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0" + before_script: + - *setup_interfaces + - echo $TSAN_OPTIONS + <<: *base_image + <<: *system_test_job + needs: + - job: clang:tsan + artifacts: true + allow_failure: true + after_script: + - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; + artifacts: + expire_in: "1 day" + paths: + - bin/tests/system/*/tsan.* + - bin/tests/system/*/*/tsan.* + - tsan/ + when: on_failure + +unit:clang:tsan: + variables: + TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER" + before_script: + - echo $TSAN_OPTIONS + - lib/isc/tests/result_test + <<: *base_image + <<: *unit_test_job + needs: + - job: clang:tsan + artifacts: true + allow_failure: true + after_script: + - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \; + artifacts: + expire_in: "1 day" + paths: + - lib/*/tests/tsan.* + - tsan/ + - kyua.log + - kyua.results + - kyua_html/ + when: on_failure + +# Jobs for mutex-based atomics on Debian SID (amd64) +mutexatomics: + variables: + CC: gcc + CFLAGS: "${CFLAGS_COMMON} -DISC_MEM_USE_INTERNAL_MALLOC=0" + EXTRA_CONFIGURE: "--with-libidn2 --enable-mutex-atomics" + <<: *base_image + <<: *build_job + +system:mutexatomics: + <<: *base_image + <<: *system_test_job + needs: + - job: mutexatomics + artifacts: true + +unit:mutexatomics: + <<: *base_image + <<: *unit_test_job + needs: + - job: mutexatomics artifacts: true # Jobs for Clang builds on Debian Stretch (amd64) clang:stretch:amd64: variables: - CC: clang + CC: ${CLANG} CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion" EXTRA_CONFIGURE: "--with-python=python3" <<: *debian_stretch_amd64_image @@ -857,38 +1059,28 @@ unit:clang:stretch:amd64: - job: clang:stretch:amd64 artifacts: true -# Jobs for Clang builds on Debian Stretch (i386) - -clang:stretch:i386: - variables: - CC: clang - CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion" - EXTRA_CONFIGURE: "--with-python=python2" - <<: *debian_stretch_i386_image - <<: *build_job - # Jobs for PKCS#11-enabled GCC builds on Debian Sid (amd64) -pkcs11:sid:amd64: +pkcs11: variables: CC: gcc CFLAGS: "${CFLAGS_COMMON}" EXTRA_CONFIGURE: "--enable-native-pkcs11 --with-pkcs11=/usr/lib/softhsm/libsofthsm2.so" - <<: *debian_sid_amd64_image + <<: *base_image <<: *build_job -system:pkcs11:sid:amd64: - <<: *debian_sid_amd64_image +system:pkcs11: + <<: *base_image <<: *system_test_job needs: - - job: pkcs11:sid:amd64 + - job: pkcs11 artifacts: true -unit:pkcs11:sid:amd64: - <<: *debian_sid_amd64_image +unit:pkcs11: + <<: *base_image <<: *unit_test_job needs: - - job: pkcs11:sid:amd64 + - job: pkcs11 artifacts: true # Jobs for Clang builds on FreeBSD 11.3 (amd64) @@ -1027,8 +1219,8 @@ system:msvc-debug:windows:amd64: # Job producing a release tarball -release:sid:amd64: - <<: *debian_sid_amd64_image +release: + <<: *base_image stage: release script: # Determine BIND version @@ -1057,7 +1249,7 @@ release:sid:amd64: # Create release tarball - tar --create --file="${CI_COMMIT_TAG}.tar.gz" --gzip release/ needs: - - job: tarball-create:sid:amd64 + - job: tarball-create artifacts: true - job: msvc:windows:amd64 artifacts: true @@ -1098,19 +1290,19 @@ release:sid:amd64: | tee curl-response.txt grep -q 'Build successfully submitted' curl-response.txt -build:coverity:sid:amd64: - <<: *debian_sid_amd64_image - stage: build +coverity: + <<: *base_image + stage: postcheck variables: CC: gcc - CFLAGS: "${CFLAGS_COMMON} -O3" + CFLAGS: "${CFLAGS_COMMON} -Og" EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2" script: - *coverity_cache_prep - *configure - *coverity_build needs: - - job: autoreconf:sid:amd64 + - job: autoreconf artifacts: true artifacts: paths: @@ -1130,12 +1322,12 @@ build:coverity:sid:amd64: # Respdiff test -respdiff:sid:amd64: - <<: *debian_sid_amd64_image +respdiff: + <<: *base_image stage: system variables: CC: gcc - CFLAGS: "${CFLAGS_COMMON} -O3" + CFLAGS: "${CFLAGS_COMMON} -Og" BIND_BASELINE_VERSION: v9_11_3 script: - ./configure --without-make-clean @@ -1149,7 +1341,7 @@ respdiff:sid:amd64: - cd ../bind-qa/bind9/respdiff - bash respdiff.sh -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}/refbind" "${CI_PROJECT_DIR}" needs: - - job: tarball-create:sid:amd64 + - job: tarball-create artifacts: true only: - tags