Adjust the GitLab CI jobs to match the new images

The custom builds (oot, asan, tsan) were mostly built using Debian sid
amd64 image.  The problem was that this image broke too easily, because
it's Debian "unstable" after all.

This commit introduces "base_image" that should be most stable with
extra bits on top (clang, coccinelle, cppcheck, ...).  Currently, that
would be Debian buster amd64.

Other changes introduced by this commit:

* Change the default clang version to 10
* Run both ASAN and TSAN with both gcc and clang compilers
* Remove Clang Debian stretch i386 job

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5fc4745190113b630665e4fa751059978148987e..fcd29898701a61ac8ce785bde28c4cf62cc4b8cb 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,9 +17,9 @@ variables:
 
   MAKE_COMMAND: make
   CONFIGURE: ./configure
-  CLANG: clang-9
-  SCAN_BUILD: scan-build-9
-  SYMBOLIZER: /usr/lib/llvm-9/bin/llvm-symbolizer
+  CLANG: clang-10
+  SCAN_BUILD: scan-build-10
+  SYMBOLIZER: /usr/lib/llvm-10/bin/llvm-symbolizer
   ASAN_SYMBOLIZER_PATH: "$SYMBOLIZER"
   CLANG_FORMAT: clang-format-10
 
@@ -148,6 +148,12 @@ stages:
   image: "$CI_REGISTRY_IMAGE:ubuntu-bionic-i386"
   <<: *linux_i386
 
+# Base image
+# This is a meta image that is used as a base for non-specific jobs
+
+.base: &base_image
+  <<: *debian_buster_amd64_image
+
 ### Job Templates
 
 .default-triggering-rules: &default_triggering_rules
@@ -168,12 +174,12 @@ stages:
 
 .precheck: &precheck_job
   <<: *default_triggering_rules
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   stage: precheck
 
 .autoconf: &autoconf_job
   <<: *release_branch_triggering_rules
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   stage: precheck
   script:
     - autoreconf -fi
@@ -207,7 +213,7 @@ stages:
     - test -z "${RUN_MAKE_INSTALL}" || make install
     - test -z "${RUN_MAKE_INSTALL}" || sh util/check-make-install
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: true
   artifacts:
     untracked: true
@@ -237,7 +243,7 @@ stages:
     - 'Set-Item -path "Env:CL" -value "/MP$([Math]::Truncate($BUILD_PARALLEL_JOBS/2))"'
     - '& msbuild.exe /maxCpuCount:2 /t:Build /p:Configuration=$VSCONF bind9.sln'
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: false
   artifacts:
     untracked: true
@@ -355,10 +361,10 @@ stages:
 
 # Jobs in the precheck stage
 
-autoreconf:sid:amd64:
+autoreconf:
   <<: *autoconf_job
 
-misc:sid:amd64:
+misc:
   <<: *precheck_job
   script:
     - sh util/check-ans-prereq.sh
@@ -386,24 +392,23 @@ misc:sid:amd64:
     expire_in: "1 day"
     when: on_failure
 
-💾:sid:amd64:
+clang-format:
   <<: *precheck_job
   needs: []
   script:
     - if [ -r .clang-format ]; then "${CLANG_FORMAT}" -i -style=file $(git ls-files '*.c' '*.h'); fi
     - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi
 
-🐞:sid:amd64:
+coccinelle:
   <<: *precheck_job
-  <<: *debian_buster_amd64_image
   needs: []
   script:
     - util/check-cocci
     - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi
 
-tarball-create:sid:amd64:
+tarball-create:
   stage: precheck
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   script:
     - source version
     - export BIND_DIRECTORY="bind-${MAJORVER}.${MINORVER}.${PATCHVER}${RELEASETYPE}${RELEASEVER}"
@@ -420,27 +425,27 @@ tarball-create:sid:amd64:
 
 # Jobs for doc builds on Debian Sid (amd64)
 
-docs:sid:amd64:
+docs:
   <<: *release_branch_triggering_rules
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   stage: docs
   script:
     - ./configure || cat config.log
     - make -C doc/misc docbook
     - make -C doc/arm Bv9ARM.html
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: true
   artifacts:
     paths:
       - doc/arm/
     expire_in: "1 month"
 
-push:docs:sid:amd64:
-  <<: *debian_sid_amd64_image
+push:docs:
+  <<: *base_image
   stage: push
   needs:
-    - job: docs:sid:amd64
+    - job: docs
       artifacts: false
   script:
     - curl -X POST -F token=$GITLAB_PAGES_DOCS_TRIGGER_TOKEN -F ref=master $GITLAB_PAGES_DOCS_TRIGGER_URL
@@ -600,9 +605,9 @@ unit:gcc:buster:amd64:
                 -o scan-build.reports \
                 make -j${BUILD_PARALLEL_JOBS:-1} all V=1
 
-scan-build:buster:amd64:
+scan-build:
   <<: *default_triggering_rules
-  <<: *debian_buster_amd64_image
+  <<: *base_image
   stage: postcheck
   variables:
     CC: "${CLANG}"
@@ -613,7 +618,7 @@ scan-build:buster:amd64:
     - *configure
     - *scan_build
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: true
   artifacts:
     paths:
@@ -647,8 +652,8 @@ unit:gcc:sid:amd64:
     - job: gcc:sid:amd64
       artifacts: true
 
-cppcheck:gcc:sid:amd64:
-  <<: *debian_sid_amd64_image
+cppcheck:
+  <<: *base_image
   <<: *cppcheck_job
   needs:
     - job: gcc:sid:amd64
@@ -656,55 +661,55 @@ cppcheck:gcc:sid:amd64:
 
 # Job for out-of-tree GCC build on Debian Sid (amd64)
 
-oot:sid:amd64:
+out-of-tree:
   variables:
     CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -O3"
+    CFLAGS: "${CFLAGS_COMMON} -Og"
     CONFIGURE: ../configure
     EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
     RUN_MAKE_INSTALL: 1
     OOT_BUILD_WORKSPACE: workspace
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *build_job
 
 # Jobs for tarball GCC builds on Debian Sid (amd64)
 
-tarball:sid:amd64:
+tarball:
   variables:
     CC: gcc
     EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
     RUN_MAKE_INSTALL: 1
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *build_job
   before_script:
     - tar --extract --file bind-*.tar.${TARBALL_EXTENSION}
     - rm -f bind-*.tar.${TARBALL_EXTENSION}
     - cd bind-*
   needs:
-    - job: tarball-create:sid:amd64
+    - job: tarball-create
       artifacts: true
   only:
     - tags
 
-system:tarball:sid:amd64:
-  <<: *debian_sid_amd64_image
+system:tarball:
+  <<: *base_image
   <<: *system_test_job
   before_script:
     - cd bind-*
     - *setup_interfaces
   needs:
-    - job: tarball:sid:amd64
+    - job: tarball
       artifacts: true
   only:
     - tags
 
-unit:tarball:sid:amd64:
-  <<: *debian_sid_amd64_image
+unit:tarball:
+  <<: *base_image
   <<: *unit_test_job
   before_script:
     - cd bind-*
   needs:
-    - job: tarball:sid:amd64
+    - job: tarball
       artifacts: true
   only:
     - tags
@@ -714,8 +719,8 @@ unit:tarball:sid:amd64:
 gcc:sid:arm64:
   variables:
     CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -O2"
-    EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
+    CFLAGS: "${CFLAGS_COMMON} -Og"
+    EXTRA_CONFIGURE: "--with-libidn2"
   <<: *debian_sid_arm64_image
   <<: *build_job
 
@@ -739,7 +744,7 @@ gcc:sid:i386:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON}"
-    EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2 --without-python"
+    EXTRA_CONFIGURE: "--with-libidn2 --without-python"
   <<: *debian_sid_i386_image
   <<: *build_job
 
@@ -853,56 +858,83 @@ unit:gcc:bionic:amd64:
     - job: gcc:bionic:amd64
       artifacts: true
 
-# Jobs for GCC builds with ASAN enabled on Debian Sid (amd64)
+# Jobs for builds with ASAN enabled
 
-asan:sid:amd64:
+gcc:asan:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined -DISC_MEM_USE_INTERNAL_MALLOC=0"
     LDFLAGS: "-fsanitize=address,undefined"
     EXTRA_CONFIGURE: "--with-libidn2"
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *build_job
 
-system:asan:sid:amd64:
+system:gcc:asan:
   variables:
     ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *system_test_job
   needs:
-    - job: asan:sid:amd64
+    - job: gcc:asan
       artifacts: true
 
-unit:asan:sid:amd64:
+unit:gcc:asan:
   variables:
     ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
-  <<: *debian_sid_amd64_image
+  <<: *base_image
+  <<: *unit_test_job
+  needs:
+    - job: gcc:asan
+      artifacts: true
+
+clang:asan:
+  variables:
+    CC: ${CLANG}
+    CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined -DISC_MEM_USE_INTERNAL_MALLOC=0"
+    LDFLAGS: "-fsanitize=address,undefined"
+    EXTRA_CONFIGURE: "--with-libidn2"
+  <<: *base_image
+  <<: *build_job
+
+system:clang:asan:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
+  <<: *base_image
+  <<: *system_test_job
+  needs:
+    - job: gcc:asan
+      artifacts: true
+
+unit:clang:asan:
+  variables:
+    ASAN_OPTIONS: ${ASAN_OPTIONS_COMMON}
+  <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: asan:sid:amd64
+    - job: gcc:asan
       artifacts: true
 
-# Jobs for GCC builds with TSAN enabled on Debian Sid (amd64)
+# Jobs for builds with TSAN enabled
 
-tsan:buster:amd64:
-  <<: *debian_buster_amd64_image
+gcc:tsan:
+  <<: *base_image
   <<: *build_job
   variables:
-    CC: "${CLANG}"
+    CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0"
     LDFLAGS: "-fsanitize=thread"
     EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock"
 
-system:tsan:buster:amd64:
+system:gcc:tsan:
   variables:
     TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
   before_script:
     - *setup_interfaces
     - echo $TSAN_OPTIONS
-  <<: *debian_buster_amd64_image
+  <<: *base_image
   <<: *system_test_job
   needs:
-    - job: tsan:buster:amd64
+    - job: gcc:tsan
       artifacts: true
   allow_failure: true
   after_script:
@@ -915,16 +947,16 @@ system:tsan:buster:amd64:
       - tsan/
     when: on_failure
 
-unit:tsan:buster:amd64:
+unit:gcc:tsan:
   variables:
     TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
   before_script:
     - echo $TSAN_OPTIONS
     - lib/isc/tests/result_test
-  <<: *debian_buster_amd64_image
+  <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: tsan:buster:amd64
+    - job: gcc:tsan
       artifacts: true
   allow_failure: true
   after_script:
@@ -939,56 +971,89 @@ unit:tsan:buster:amd64:
       - kyua_html/
     when: on_failure
 
-rwlock:sid:amd64:
+clang:tsan:
+  <<: *base_image
+  <<: *build_job
   variables:
-    CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -Wall -Wextra -O2 -g -DISC_MEM_USE_INTERNAL_MALLOC=0"
+    CC: "${CLANG}"
+    CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread -DISC_MEM_USE_INTERNAL_MALLOC=0"
+    LDFLAGS: "-fsanitize=thread"
     EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock"
-  <<: *debian_sid_amd64_image
-  <<: *build_job
 
-system:rwlock:sid:amd64:
-  <<: *debian_sid_amd64_image
+system:clang:tsan:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER exitcode=0"
+  before_script:
+    - *setup_interfaces
+    - echo $TSAN_OPTIONS
+  <<: *base_image
   <<: *system_test_job
   needs:
-    - job: rwlock:sid:amd64
+    - job: clang:tsan
       artifacts: true
+  allow_failure: true
+  after_script:
+    - find bin -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - bin/tests/system/*/tsan.*
+      - bin/tests/system/*/*/tsan.*
+      - tsan/
+    when: on_failure
 
-unit:rwlock:sid:amd64:
-  <<: *debian_sid_amd64_image
+unit:clang:tsan:
+  variables:
+    TSAN_OPTIONS: "second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=$SYMBOLIZER"
+  before_script:
+    - echo $TSAN_OPTIONS
+    - lib/isc/tests/result_test
+  <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: rwlock:sid:amd64
+    - job: clang:tsan
       artifacts: true
+  allow_failure: true
+  after_script:
+    - find lib -name 'tsan.*' -exec python3 util/parse_tsan.py {} \;
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - lib/*/tests/tsan.*
+      - tsan/
+      - kyua.log
+      - kyua.results
+      - kyua_html/
+    when: on_failure
 
 # Jobs for mutex-based atomics on Debian SID (amd64)
-mutexatomics:sid:amd64:
+mutexatomics:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON} -DISC_MEM_USE_INTERNAL_MALLOC=0"
     EXTRA_CONFIGURE: "--with-libidn2 --enable-mutex-atomics"
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *build_job
 
-system:mutexatomics:sid:amd64:
-  <<: *debian_sid_amd64_image
+system:mutexatomics:
+  <<: *base_image
   <<: *system_test_job
   needs:
-    - job: mutexatomics:sid:amd64
+    - job: mutexatomics
       artifacts: true
 
-unit:mutexatomics:sid:amd64:
-  <<: *debian_sid_amd64_image
+unit:mutexatomics:
+  <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: mutexatomics:sid:amd64
+    - job: mutexatomics
       artifacts: true
 
 # Jobs for Clang builds on Debian Stretch (amd64)
 
 clang:stretch:amd64:
   variables:
-    CC: clang
+    CC: ${CLANG}
     CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion"
     EXTRA_CONFIGURE: "--with-python=python3"
   <<: *debian_stretch_amd64_image
@@ -1001,38 +1066,28 @@ unit:clang:stretch:amd64:
     - job: clang:stretch:amd64
       artifacts: true
 
-# Jobs for Clang builds on Debian Stretch (i386)
-
-clang:stretch:i386:
-  variables:
-    CC: clang
-    CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion"
-    EXTRA_CONFIGURE: "--with-python=python2"
-  <<: *debian_stretch_i386_image
-  <<: *build_job
-
 # Jobs for PKCS#11-enabled GCC builds on Debian Sid (amd64)
 
-pkcs11:sid:amd64:
+pkcs11:
   variables:
     CC: gcc
     CFLAGS: "${CFLAGS_COMMON}"
     EXTRA_CONFIGURE: "--enable-native-pkcs11 --with-pkcs11=/usr/lib/softhsm/libsofthsm2.so"
-  <<: *debian_sid_amd64_image
+  <<: *base_image
   <<: *build_job
 
-system:pkcs11:sid:amd64:
-  <<: *debian_sid_amd64_image
+system:pkcs11:
+  <<: *base_image
   <<: *system_test_job
   needs:
-    - job: pkcs11:sid:amd64
+    - job: pkcs11
       artifacts: true
 
-unit:pkcs11:sid:amd64:
-  <<: *debian_sid_amd64_image
+unit:pkcs11:
+  <<: *base_image
   <<: *unit_test_job
   needs:
-    - job: pkcs11:sid:amd64
+    - job: pkcs11
       artifacts: true
 
 # Jobs for Clang builds on FreeBSD 11.3 (amd64)
@@ -1144,8 +1199,8 @@ system:msvc-debug:windows:amd64:
 
 # Job producing a release tarball
 
-release:sid:amd64:
-  <<: *debian_sid_amd64_image
+release:
+  <<: *base_image
   stage: release
   script:
     # Determine BIND version
@@ -1174,7 +1229,7 @@ release:sid:amd64:
     # Create release tarball
     - tar --create --file="${CI_COMMIT_TAG}.tar.gz" --gzip release/
   needs:
-    - job: tarball-create:sid:amd64
+    - job: tarball-create
       artifacts: true
     - job: msvc:windows:amd64
       artifacts: true
@@ -1215,19 +1270,19 @@ release:sid:amd64:
        | tee curl-response.txt
   grep -q 'Build successfully submitted' curl-response.txt
 
-build:coverity:sid:amd64:
-  <<: *debian_sid_amd64_image
-  stage: build
+coverity:
+  <<: *base_image
+  stage: postcheck
   variables:
     CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -O3"
+    CFLAGS: "${CFLAGS_COMMON} -Og"
     EXTRA_CONFIGURE: "--enable-dnstap --with-libidn2"
   script:
     - *coverity_cache_prep
     - *configure
     - *coverity_build
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: true
   artifacts:
     paths:
@@ -1247,12 +1302,12 @@ build:coverity:sid:amd64:
 
 # Respdiff test
 
-respdiff:sid:amd64:
-  <<: *debian_sid_amd64_image
+respdiff:
+  <<: *base_image
   stage: system
   variables:
     CC: gcc
-    CFLAGS: "${CFLAGS_COMMON} -O3"
+    CFLAGS: "${CFLAGS_COMMON} -Og"
     BIND_BASELINE_VERSION: v9_11_3
   script:
     - ./configure --without-make-clean
@@ -1266,7 +1321,7 @@ respdiff:sid:amd64:
     - cd ../bind-qa/bind9/respdiff
     - bash respdiff.sh -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}/refbind" "${CI_PROJECT_DIR}"
   needs:
-    - job: tarball-create:sid:amd64
+    - job: tarball-create
       artifacts: true
   only:
     - tags
@@ -1279,11 +1334,11 @@ respdiff:sid:amd64:
 
 # ABI check
 
-abi-check:sid:amd64:
-  <<: *debian_sid_amd64_image
+abi-check:
+  <<: *base_image
   stage: build
   needs:
-    - job: autoreconf:sid:amd64
+    - job: autoreconf
       artifacts: true
   variables:
     CC: gcc