From 15c1353544be799752b68fb891aefd9c589a86fa Mon Sep 17 00:00:00 2001 From: Tomas Krizek Date: Tue, 21 Dec 2021 16:38:45 +0100 Subject: [PATCH] pytests: migrate to LXC runner Due to missing support on some of the regular runners, let's migrate these tests to our special LXC runners. This should hopefully make the results more reliable and stable. The downside is that we have to keep an additional image (and recipe) for LXC, since it' slightly different. However, it's probably worth it, since we'll likely migrate some other tests there in the future (for better stability). --- .gitlab-ci.yml | 16 +++- ci/images/lxc-debian-11/Dockerfile | 131 +++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 ci/images/lxc-debian-11/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d313385af..bd8d84697 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -346,10 +346,18 @@ pkgtest: - $SKIP_CI == "1" pytests: - <<: *test_flaky - needs: - - build-asan - allow_failure: true # due to issues with runners: OSError 99 + # these are executed on LXC runners to increase stability + image: $CI_REGISTRY/knot/knot-resolver/ci/lxc-debian-11:knot-$KNOT_VERSION + needs: [] + tags: + - lxc + - amd64 + before_script: + # build-asan artifacts can't be reused (different container is used) + - CC=clang CXX=clang++ CFLAGS=-fno-sanitize-recover=all CXXFLAGS=-fno-sanitize=undefined meson build_ci_asan --default-library=static --prefix=$PREFIX -Db_sanitize=address,undefined -Dextra_tests=enabled + - ninja -C build_ci_asan + - ninja -C build_ci_asan install >/dev/null + # END lxc specific section artifacts: when: always paths: diff --git a/ci/images/lxc-debian-11/Dockerfile b/ci/images/lxc-debian-11/Dockerfile new file mode 100644 index 000000000..32c6bfe66 --- /dev/null +++ b/ci/images/lxc-debian-11/Dockerfile @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +FROM registry.nic.cz/labs/lxc-gitlab-runner/debian-11:latest +MAINTAINER Knot Resolver +# >= 3.0 needed because of --enable-xdp=yes +ARG KNOT_BRANCH=3.1 +ENV DEBIAN_FRONTEND=noninteractive + +# generic cleanup +RUN apt-get update -qq + +# Knot and Knot Resolver dependencies +RUN apt-get install -y -qqq git make cmake pkg-config meson \ + build-essential bsdmainutils libtool autoconf libcmocka-dev \ + liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \ + libelf-dev libmnl-dev libidn11-dev libuv1-dev \ + libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev + +# Build and testing deps for Resolver's dnstap module (go stuff is just for testing) +RUN apt-get install -y -qqq \ + protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \ + golang-any +RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap,golang/protobuf/proto}" + +# documentation dependencies +RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme + +# Python packages required for Deckard CI +# Python: grab latest versions from PyPi +# (Augeas binding in Debian packages are slow and buggy) +RUN apt-get install -y -qqq python3-pip wget augeas-tools +RUN pip3 install --upgrade pip +RUN pip3 install pylint +RUN pip3 install pep8 +RUN pip3 install pytest-xdist +# FIXME replace with dnspython >= 2.2.0 once released +RUN pip3 install git+git://github.com/bwelling/dnspython.git@72348d4698a8f8b209fbdf9e72738904ad31b930 +# tests/pytest dependencies: skip over broken versions +RUN pip3 install jinja2 'pytest != 6.0.0' pytest-html pytest-xdist +# apkg for packaging +RUN pip3 install apkg + +# packet capture tools for Deckard +RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common + +# Faketime for Deckard +RUN apt-get install -y -qqq faketime + +# C dependencies for python-augeas +RUN apt-get install -y -qqq libaugeas-dev libffi-dev +# Python dependencies for Deckard +RUN wget https://gitlab.nic.cz/knot/deckard/raw/master/requirements.txt -O /tmp/deckard-req.txt +RUN pip3 install -r /tmp/deckard-req.txt + +# build and install latest version of Knot DNS +RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot +WORKDIR /tmp/knot +RUN pwd +RUN autoreconf -if +RUN ./configure --prefix=/usr --enable-xdp=yes +RUN CFLAGS="-g" make +RUN make install +RUN ldconfig + +# Valgrind for kresd CI +RUN apt-get install valgrind -y -qqq +RUN wget https://github.com/LuaJIT/LuaJIT/raw/v2.1.0-beta3/src/lj.supp -O /lj.supp +# TODO: rebuild LuaJIT with Valgrind support + +# Lua lint for kresd CI +RUN apt-get install luarocks -y -qqq +RUN luarocks --lua-version 5.1 install luacheck + +# respdiff for kresd CI +RUN apt-get install lmdb-utils -y -qqq +RUN git clone --depth=1 https://gitlab.nic.cz/knot/respdiff /var/opt/respdiff +RUN pip3 install -r /var/opt/respdiff/requirements.txt + +# Python static analysis for respdiff +RUN pip3 install mypy +RUN pip3 install flake8 + +# Python requests for CI scripts +RUN pip3 install requests + +# docker-py for packaging tests +RUN pip3 install docker + +# Unbound for respdiff +RUN apt-get install unbound unbound-anchor -y -qqq +RUN printf "server:\n interface: 127.0.0.1@53535\n use-syslog: yes\n do-ip6: no\nremote-control:\n control-enable: no\n" >> /etc/unbound/unbound.conf + +# BIND for respdiff +RUN apt-get install bind9 -y -qqq +RUN printf '\nOPTIONS="-4 $OPTIONS"' >> /etc/default/bind9 +RUN printf 'options {\n directory "/var/cache/bind";\n listen-on port 53533 { 127.0.0.1; };\n listen-on-v6 port 53533 { ::1; };\n};\n' > /etc/bind/named.conf.options + +# PowerDNS Recursor for Deckard CI +RUN apt-get install pdns-recursor -y -qqq + +# code coverage +RUN apt-get install -y -qqq lcov +RUN luarocks --lua-version 5.1 install luacov + +# LuaJIT binary for stand-alone scripting +RUN apt-get install -y -qqq luajit + +# clang for kresd CI, version updated as debian updates it +RUN apt-get install -y -qqq clang clang-tools clang-tidy + +# OpenBuildService CLI tool +RUN apt-get install -y osc + +# curl (API) +RUN apt-get install -y curl + +# configure knot-resolver-testing OBS repo for dependencies missing in Debian +RUN echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/knot-resolver-testing/Debian_11/ /' > /etc/apt/sources.list.d/knot-resolver-testing.list +RUN wget -nv https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-testing/Debian_11/Release.key -O Release.key +RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add Release.key +RUN rm Release.key +RUN apt-get update -qq + +# packages from our knot-resolver-testing repo +RUN apt-get update +RUN apt-get install -y -qqq lua-psl + +# en_US.UTF-8 locale for scripts.update-authors.sh +RUN apt-get install -y -qqq locales +RUN sed -i "/en_US.UTF-8/ s/^#\(.*\)/\1/" /etc/locale.gen +RUN locale-gen -- 2.47.2