From: Erik Winkels Date: Wed, 3 Feb 2021 21:18:24 +0000 (+0100) Subject: Python version of repo test script. X-Git-Tag: dnsdist-1.6.0-alpha2~38^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4d43fb08046257ce264f5a74bae6e830dd539d55;p=thirdparty%2Fpdns.git Python version of repo test script. This replaces the Bourne shell version. --- diff --git a/build-scripts/docker/generate-repo-files.sh b/build-scripts/docker/generate-repo-files.sh deleted file mode 100755 index 6aee76fb2c..0000000000 --- a/build-scripts/docker/generate-repo-files.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/bin/sh -# -# - `docker build --no-cache --pull --file Dockerfile.auth-41.ubuntu-bionic --tag auth-41.ubuntu-bionic .` -# - `docker run -it auth-41.ubuntu-bionic` -# - `docker run -it auth-41.ubuntu-bionic /bin/bash` -# - `dnsdist --verbose 9.9.9.9` -# - `pdns_recursor` -# - `pdns_server` -# -# Remi contributed this snippet: -# -# #!/bin/bash -# -# readonly product=dnsdist-15 -# -# for version in centos-6 centos-7 centos-8 debian-buster debian-stretch ubuntu-bionic ubuntu-xenial; do -# docker build --no-cache --pull --file Dockerfile.${product}.${version} --tag ${product}.${version} . -# done -# -# for version in centos-6 centos-7 centos-8 debian-buster debian-stretch ubuntu-bionic ubuntu-xenial; do -# docker run -it ${product}.${version} dnsdist -v 9.9.9.9 -# done -# -# For Raspbian you need to have installed: -# - Void Linux: `qemu`, `qemu-user-static` and `binfmt-support` - -if [ "$1" = "" -o "$1" = "-?" -o "$1" = "-h" -o "$1" = "--help" ]; then - echo "Usage: generate-repo-files.sh RELEASE" - echo - echo " • RELEASE: [ auth-41 | auth-42 | auth-43 | auth-44 | auth-master |" - echo " rec-42 | rec-43 | rec-44 | rec-45 | rec-master |" - echo " dnsdist-15 | dnsdist-16 | dnsdist-master ]" - exit 1 -fi - - -write_centos() -{ - OS=centos - VERSION=$1 - PKG=$2 - CMD=$3 - - if [ "$VERSION" = "8" ]; then - CENTOS8_FLAGS="--nobest" - else - CENTOS8_FLAGS="" - fi - - cat < Dockerfile.$RELEASE.$OS-$VERSION -FROM $OS:$VERSION - -RUN yum install -y epel-release bind-utils -EOF - - if [ "$VERSION" = "7" ]; then - cat <> Dockerfile.$RELEASE.$OS-$VERSION -RUN yum install -y yum-plugin-priorities -EOF - elif [ "$RELEASE" = "dnsdist-15" -a "$VERSION" = "8" ]; then - cat <> Dockerfile.$RELEASE.$OS-$VERSION -RUN dnf install -y 'dnf-command(config-manager)' -RUN dnf config-manager --set-enabled powertools -EOF - fi - - cat <> Dockerfile.$RELEASE.$OS-$VERSION -RUN curl -o /etc/yum.repos.d/powerdns-$RELEASE.repo https://repo.powerdns.com/repo-files/$OS-$RELEASE.repo -RUN yum install --assumeyes $CENTOS8_FLAGS $PKG -EOF - - if [ "$RELEASE" = "rec-43" -o "$RELEASE" = "rec-44" -o "$RELEASE" = "rec-45" ]; then - cat <> Dockerfile.$RELEASE.$OS-$VERSION - -RUN mkdir /var/run/pdns-recursor -EOF - fi - - cat <> Dockerfile.$RELEASE.$OS-$VERSION - -CMD $CMD --version -EOF -} - - -write_debian_or_ubuntu() -{ - OS=$1 - VERSION=$2 - PKG=$3 - CMD=$4 - - ARCHSTRING='[arch=amd64]' - - OSIMG=$OS - if [ "$OS" = "raspbian" ]; then - OSIMG=resin/rpi-raspbian - ARCHSTRING='' - fi - - cat < pdns.list.$RELEASE.$OS-$VERSION -deb $ARCHSTRING http://repo.powerdns.com/$OS $VERSION-$RELEASE main -EOF - - # For the following two maybe only create depending on package, but - # it's not really a big deal. - - # if not exists - cat < dnsdist.debian-and-ubuntu -Package: dnsdist* -Pin: origin repo.powerdns.com -Pin-Priority: 600 -EOF - - # if not exists - cat < pdns.debian-and-ubuntu -Package: pdns-* -Pin: origin repo.powerdns.com -Pin-Priority: 600 -EOF - - cat < Dockerfile.$RELEASE.$OS-$VERSION -FROM $OSIMG:$VERSION - -RUN apt-get update -RUN apt-get install -y curl gnupg dnsutils apt-transport-https - -COPY dnsdist.debian-and-ubuntu /etc/apt/preferences.d/dnsdist -COPY pdns.debian-and-ubuntu /etc/apt/preferences.d/pdns -COPY pdns.list.$RELEASE.$OS-$VERSION /etc/apt/sources.list.d/pdns.list - -RUN curl https://repo.powerdns.com/FD380FBB-pub.asc | apt-key add - -RUN curl https://repo.powerdns.com/CBC8B383-pub.asc | apt-key add - -RUN apt-get update -RUN apt-get install -y $PKG -EOF - - if [ "$RELEASE" = "rec-43" -o "$RELEASE" = "rec-44" -o "$RELEASE" = "rec-45" ]; then - cat <> Dockerfile.$RELEASE.$OS-$VERSION - -RUN mkdir /var/run/pdns-recursor -EOF - fi - - cat <> Dockerfile.$RELEASE.$OS-$VERSION - -CMD $CMD --version -EOF -} - - -write_debian() -{ - write_debian_or_ubuntu debian $1 $2 $3 - write_debian_or_ubuntu raspbian $1 $2 $3 -} - - -write_ubuntu() -{ - write_debian_or_ubuntu ubuntu $1 $2 $3 -} - - -RELEASE=$1 - -# It would be smarter to list the supported products per distro by now. -if [ "$RELEASE" = "auth-41" ]; then - write_centos 7 pdns pdns_server - write_ubuntu bionic pdns-server pdns_server -elif [ "$RELEASE" = "auth-42" ]; then - write_centos 7 pdns pdns_server - write_centos 8 pdns pdns_server - write_debian buster pdns-server pdns_server - write_ubuntu bionic pdns-server pdns_server -elif [ "$RELEASE" = "auth-43" -o "$RELEASE" = "auth-44" -o "$RELEASE" = "auth-master" ]; then - if [ "$RELEASE" != "auth-44" ]; then write_centos 6 pdns pdns_server; fi - write_centos 7 pdns pdns_server - write_centos 8 pdns pdns_server - write_debian buster pdns-server pdns_server - write_ubuntu bionic pdns-server pdns_server - write_ubuntu focal pdns-server pdns_server -elif [ "$RELEASE" = "rec-42" ]; then - write_centos 7 pdns-recursor pdns_recursor - write_centos 8 pdns-recursor pdns_recursor - write_debian buster pdns-recursor pdns_recursor - write_ubuntu bionic pdns-recursor pdns_recursor -elif [ "$RELEASE" = "rec-43" ]; then - write_centos 7 pdns-recursor pdns_recursor - write_centos 8 pdns-recursor pdns_recursor - write_debian buster pdns-recursor pdns_recursor - write_ubuntu bionic pdns-recursor pdns_recursor - write_ubuntu focal pdns-recursor pdns_recursor -elif [ "$RELEASE" = "rec-44" -o "$RELEASE" = "rec-45" -o "$RELEASE" = "rec-master" ]; then - write_centos 7 pdns-recursor pdns_recursor - write_centos 8 pdns-recursor pdns_recursor - write_debian buster pdns-recursor pdns_recursor - write_ubuntu bionic pdns-recursor pdns_recursor - write_ubuntu focal pdns-recursor pdns_recursor -elif [ "$RELEASE" = "dnsdist-15" -o "$RELEASE" = "dnsdist-16" -o "$RELEASE" = "dnsdist-master" ]; then - write_centos 7 dnsdist dnsdist - write_centos 8 dnsdist dnsdist - write_debian buster dnsdist dnsdist - write_ubuntu bionic dnsdist dnsdist - write_ubuntu focal dnsdist dnsdist -else - echo "Invalid release: $RELEASE" - exit 1 -fi diff --git a/build-scripts/docker/repo-test/.gitignore b/build-scripts/docker/repo-test/.gitignore new file mode 100644 index 0000000000..a17800c4fd --- /dev/null +++ b/build-scripts/docker/repo-test/.gitignore @@ -0,0 +1,4 @@ +venv/ +Dockerfile.* +pdns.list.* +pkg-pin diff --git a/build-scripts/docker/repo-test/README.md b/build-scripts/docker/repo-test/README.md new file mode 100644 index 0000000000..36038b437b --- /dev/null +++ b/build-scripts/docker/repo-test/README.md @@ -0,0 +1,79 @@ +# Tool for Testing PowerDNS Packages & Repositories + +This directory contains the `generate-repo-files.py` script which can +generate all the required files to build and run a Docker image which +can download and run the latest version of a PowerDNS Authoritative +Server, Recursor or DNSDist release. + +To see the supported releases do `./generate-repo-files.py --help`. + +This tool is mainly used internally to test releases but might be useful +for others. + +## Dependencies + +- Python 3 + - Jinja2 + +## Usage + +The steps assume you just want to run the script and have no experience +with Python. If you do you can evaluate for yourself whether you need +some of these steps. + +In the directory of this README and `generate-repo-files.py` file do: + +- `python3 -m venv venv` +- `bash` +- `source venv/bin/activate` +- `pip install --upgrade pip` +- `pip install -r requirements.txt` +- `./generate-repo-files.py rec-45` + - where `rec-45` is an example release, do + `./generate-repo-files.py --help` to see all supported releases + - do `./generate-repo-files.py --test rec-45` to also test the + release + +This will create files like `Dockerfile.rec-45.centos-7` and some +additional support file like `pdns.list.rec-45.*` and `pkg-pin`. + +The Docker file can then be used to build an image: `docker build --no-cache --pull --file Dockerfile.rec-45.centos-7 --tag rec-45.centos-7 .` + +And this image can be run with `docker run -it rec-45.centos-7` which +just runs `pdns_recursor --version`. + +The image can be entered with `docker run -it rec-45.centos-7 /bin/bash` +where you can do `pdns_recursor` or whatever you like. + +## Elaborations + +### `--test` + +This also tests the release by using the generated Docker files to build +an image and run a container using that image. + +### `--run-output` + +This argument can be a little unclear. It is only applicable when +`--test` is also supplied and `--verbose` is *NOT* supplied, otherwise +it is ignored. + +Since run output is not a lot when testing releases it can be nice to +show the output from running the container even when `--verbose` is off +(where it can be drowned out in the verbose output). + +## To Do + +- medium priority: + - make not using `--no-cache` and `--pull` for building a Docker + image an option, currently these are always used +- low priority: + - option to test a specific release version and not just the latest +- maybe: + - make error codes we get for building and running containers + readable if we can find a lib for this, otherwise it requires too + much maintenance + - `errno` http://joeyh.name/code/moreutils/ + - parameter to test all the things! (this can currently easily done + by a shell script / command) + - `for RELEASE in auth-42 auth-43 auth-44 auth-master rec-42 rec-43 rec-44 rec-45 rec-master dnsdist-15 dnsdist-16 dnsdist-master; do ./generate-repo-files.py --test $RELEASE; done` diff --git a/build-scripts/docker/repo-test/generate-repo-files.py b/build-scripts/docker/repo-test/generate-repo-files.py new file mode 100755 index 0000000000..1aba3b3cbb --- /dev/null +++ b/build-scripts/docker/repo-test/generate-repo-files.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# +# Given Python's versioning history I'm going with `python3`. +# +# Usage: +# - `python3 -m venv venv` +# - `bash` +# - `source venv/bin/activate` +# - `pip install --upgrade pip` +# - `pip install -r requirements.txt` +# - `./generate-repo-files.py auth-41` + +# Modules + +import argparse +import subprocess +import sys + +from pathlib import Path + +# since we use this at OX (or Ansible uses it, whatever) +from jinja2 import Environment, FileSystemLoader + + +# Globals + +g_version = '0.0.1' + +g_verbose = False + +g_env = Environment( + loader=FileSystemLoader('templates/') +) + +g_dockerfile = 'Dockerfile.' +g_run_output = False + + +# Init Functions + +def init_argparser(): + parser = argparse.ArgumentParser(description='Generate Docker files to ' + + 'test PowerDNS repositories.') + parser.add_argument('release', metavar='RELEASE', + choices=[# Authoritative Server + 'auth-42', 'auth-43', 'auth-44', 'auth-master', + # Recursor + 'rec-42', 'rec-43', 'rec-44', 'rec-45', + 'rec-master', + # DNSDist + 'dnsdist-15', 'dnsdist-16', 'dnsdist-master' + ], + help='the release to generate Docker files for: ' + + '%(choices)s') + parser.add_argument('--run-output', action='store_true', + help='always show output from running a container') + parser.add_argument('--test', action='store_true', + help='test the release') + parser.add_argument('--verbose', action='store_true', + help='verbose output') + parser.add_argument('--version', action='store_true', + help='print version') + return parser + + +# Release File Functions + +def write_dockerfile (os, os_version, release): + tpl = g_env.get_template('Dockerfile-{}.jinja2'.format(os)) + + if os == 'raspbian': + os_image = 'resin/rpi-raspbian' + else: + os_image = os + + if release.startswith('auth-'): + if os == 'centos': + pkg = 'pdns' + else: + pkg = 'pdns-server' + cmd = 'pdns_server' + elif release.startswith('rec-'): + pkg = 'pdns-recursor' + cmd = 'pdns_recursor' + elif release.startswith('dnsdist-'): + pkg = 'dnsdist' + cmd = 'dnsdist' + + f = open('{}{}.{}-{}'.format(g_dockerfile, release, os, os_version), 'w') + + # This comment was in the template for the `--nobest` part but that makes + # the template look even more different than the final output, so: + # + # > When should the logic be in the code and when in the template? :shrug: + # > I prefer it to be in the code but I also do not want to add extra vars + # > and logic to the code unless necessary. + f.write(tpl.render({ "os": os, + "os_image": os_image, + "os_version": os_version, + "release": release, + "cmd": cmd, + "pkg": pkg })) + f.close() + + +def write_list_file (os, os_version, release): + tpl = g_env.get_template('pdns-list.jinja2') + + if os in ['debian', 'ubuntu']: + arch = ' [arch=amd64] ' + else: + arch = ' ' + + f = open('pdns.list.{}.{}-{}'.format(release, os, os_version), 'w') + f.write(tpl.render({ "os": os, + "os_version": os_version, + "release": release, + "arch": arch })) + f.close() + + +def write_pkg_pin_file (release): + tpl = g_env.get_template('pkg-pin.jinja2') + + if release.startswith('auth-') or release.startswith('rec-'): + pkg = 'pdns-' + elif release.startswith('dnsdist-'): + pkg = 'dnsdist' + + f = open('pkg-pin', 'w') + f.write(tpl.render({ "pkg": pkg })) + f.close() + + +def write_release_files (release): + if g_verbose: + print("Writing release files...") + + if release in ['auth-43', 'auth-master']: + write_dockerfile('centos', '6', release) + + if release in ['auth-41', 'auth-42', 'auth-43', 'auth-44', 'auth-master', + 'rec-42', 'rec-43', 'rec-44', 'rec-45', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + write_dockerfile('centos', '7', release) + write_dockerfile('ubuntu', 'bionic', release) + write_list_file('ubuntu', 'bionic', release) + write_pkg_pin_file(release) + + if release in ['auth-42', 'auth-43', 'auth-44', 'auth-master', + 'rec-42', 'rec-43', 'rec-44', 'rec-45', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + write_dockerfile('centos', '8', release) + write_dockerfile('debian', 'buster', release) + write_dockerfile('raspbian', 'buster', release) + write_list_file('debian', 'buster', release) + write_list_file('raspbian', 'buster', release) + + if release in ['auth-43', 'auth-44', 'auth-master', + 'rec-43', 'rec-44', 'rec-45', 'rec-master', + 'dnsdist-15', 'dnsdist-16', 'dnsdist-master']: + write_dockerfile('ubuntu', 'focal', release) + write_list_file('ubuntu', 'focal', release) + + +# Test Release Functions + +def build (dockerfile): + # Maybe create `determine_tag` function. + if len(str(dockerfile)) <= len(g_dockerfile): + print('Unable to determine tag for {}'.format(dockerfile)) + return (None, None) + tag = str(dockerfile)[len(g_dockerfile):] + print('Building Docker image using {}...'.format(dockerfile)) + if g_verbose: + print(' - tag = {}'.format(tag)) + cp = subprocess.run(['docker', 'build', '--no-cache', '--pull', '--file', + dockerfile, '--tag', tag, '.'], + capture_output=not(g_verbose)) + # FIXME write failed output to log + if cp.returncode != 0: + print('Error building {}: {}'.format(tag, repr(cp.returncode))) + return ( tag, cp.returncode ) + return ( tag, cp.returncode ) + + +def run (tag): + if g_run_output: + capture_run_output = False + else: + capture_run_output = not(g_verbose) + print('Running Docker container tagged {}...'.format(tag)) + cp = subprocess.run(['docker', 'run', tag], + capture_output=capture_run_output) + # for some reason 99 is returned on `cmd --version` :shrug: + if cp.returncode != 0 and cp.returncode != 99: + # FIXME write failed output to log + print('Error running {}: {}'.format(tag, repr(cp.returncode))) + return cp.returncode + return cp.returncode + + +def collect_dockerfiles (release): + if g_verbose: + print('Collecting release files for {}...'.format(release)) + p = Path('.') + files = list(p.glob('{}{}.*'.format(g_dockerfile, release))) + if g_verbose: + for file in files: + print(' - {}'.format(file)) + return files + + +def test_release (release): + # sorted because we want determinism + dockerfiles = sorted(collect_dockerfiles(release)) + failed_builds = [] + failed_runs = [] + print('=== testing {} ==='.format(release)) + for df in dockerfiles: + if g_verbose: + print('--- {} ---'.format(df)) + (tag, returncode) = build(df) + if returncode != 0: + print('Skipping running {} due to build error: {}' + .format(df, returncode)) + failed_builds.append((str(df), returncode)) + elif tag is None: + print('Skipping running {} due to undetermined tag.'.format(df)) + failed_builds.append((str(df), returncode)) + else: + returncode = run(tag) + # for some reason 99 is returned on `cmd --version` :shrug: + if returncode != 0 and returncode != 99: + failed_runs.append((tag, returncode)) + print('Test done.') + if len(failed_builds) > 0: + print('- failed builds:') + for fb in failed_builds: + print(' - {}'.format(fb)) + if len(failed_runs) > 0: + print('- failed runs:') + for fr in failed_runs: + print(' - {}'.format(fr)) + + +# Main Program + +parser = init_argparser() +args = parser.parse_args() + +if args.version: + print('generate-repo-files v' + g_version) + sys.exit(0) + +if args.verbose: + g_verbose = True + +if args.run_output: + g_run_output = True + +write_release_files(args.release) + +if args.test: + test_release(args.release) diff --git a/build-scripts/docker/repo-test/requirements.txt b/build-scripts/docker/repo-test/requirements.txt new file mode 100644 index 0000000000..7f7afbf3bf --- /dev/null +++ b/build-scripts/docker/repo-test/requirements.txt @@ -0,0 +1 @@ +jinja2 diff --git a/build-scripts/docker/repo-test/templates/Dockerfile-centos.jinja2 b/build-scripts/docker/repo-test/templates/Dockerfile-centos.jinja2 new file mode 100644 index 0000000000..69ea0576d3 --- /dev/null +++ b/build-scripts/docker/repo-test/templates/Dockerfile-centos.jinja2 @@ -0,0 +1,22 @@ +FROM {{ os_image }}:{{ os_version }} + +RUN yum install -y epel-release bind-utils + +{% if os_version == '7' %} +RUN yum install -y yum-plugin-priorities +{% endif %} + +{% if release == 'dnsdist-15' and os_version == '8' %} +RUN dnf install -y 'dnf-command(config-manager)' +RUN dnf config-manager --set-enabled powertools +{% endif %} + +RUN curl -o /etc/yum.repos.d/powerdns-{{ release }}.repo https://repo.powerdns.com/repo-files/{{ os }}-{{ release }}.repo +RUN yum install --assumeyes {%- if os_version == '8' %} --nobest{% endif %} {{ pkg }} + +{% if release.startswith('rec-') %} +RUN mkdir /var/run/pdns-recursor +{% endif %} + +CMD {{ cmd }} --version + diff --git a/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 b/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 new file mode 100644 index 0000000000..fea16594d5 --- /dev/null +++ b/build-scripts/docker/repo-test/templates/Dockerfile-debian.jinja2 @@ -0,0 +1,25 @@ +FROM {{ os_image }}:{{ os_version }} + +RUN apt-get update +RUN apt-get install -y curl gnupg dnsutils apt-transport-https + +{% if release.startswith('dnsdist-') %} +COPY pkg-pin /etc/apt/preferences.d/dnsdist +{% else %} +COPY pkg-pin /etc/apt/preferences.d/pdns +{% endif %} + +COPY pdns.list.{{ release }}.{{ os }}-{{ os_version }} /etc/apt/sources.list.d/pdns.list + +RUN curl https://repo.powerdns.com/FD380FBB-pub.asc | apt-key add - +RUN curl https://repo.powerdns.com/CBC8B383-pub.asc | apt-key add - +RUN apt-get update +RUN apt-get install -y {{ pkg }} + +{# in the old script this was just for rec-43, -44 and -45 #} +{% if release.startswith('rec-') %} +RUN mkdir /var/run/pdns-recursor +{% endif %} + +CMD {{ cmd }} --version + diff --git a/build-scripts/docker/repo-test/templates/Dockerfile-raspbian.jinja2 b/build-scripts/docker/repo-test/templates/Dockerfile-raspbian.jinja2 new file mode 120000 index 0000000000..95f0fd1fc6 --- /dev/null +++ b/build-scripts/docker/repo-test/templates/Dockerfile-raspbian.jinja2 @@ -0,0 +1 @@ +Dockerfile-debian.jinja2 \ No newline at end of file diff --git a/build-scripts/docker/repo-test/templates/Dockerfile-ubuntu.jinja2 b/build-scripts/docker/repo-test/templates/Dockerfile-ubuntu.jinja2 new file mode 120000 index 0000000000..95f0fd1fc6 --- /dev/null +++ b/build-scripts/docker/repo-test/templates/Dockerfile-ubuntu.jinja2 @@ -0,0 +1 @@ +Dockerfile-debian.jinja2 \ No newline at end of file diff --git a/build-scripts/docker/repo-test/templates/pdns-list.jinja2 b/build-scripts/docker/repo-test/templates/pdns-list.jinja2 new file mode 100644 index 0000000000..212150189e --- /dev/null +++ b/build-scripts/docker/repo-test/templates/pdns-list.jinja2 @@ -0,0 +1,2 @@ +deb {{- arch -}} http://repo.powerdns.com/{{ os }} {{ os_version }}-{{ release }} main + diff --git a/build-scripts/docker/repo-test/templates/pkg-pin.jinja2 b/build-scripts/docker/repo-test/templates/pkg-pin.jinja2 new file mode 100644 index 0000000000..f526a8d2ae --- /dev/null +++ b/build-scripts/docker/repo-test/templates/pkg-pin.jinja2 @@ -0,0 +1,4 @@ +Package: {{ pkg }}* +Pin: origin repo.powerdns.com +Pin-Priority: 600 +