+++ /dev/null
-#!/usr/bin/env python3
-# SPDX-License-Identifier: GPL-3.0-or-later
-'''
-Generate minimal Dockefile to build, install, run, and test kresd and modules.
-
-It merges data from two sources:
-
-1. Distribution specific commands for package installation etc.
- These come from distros/ subtree with two-level hierarchy:
- <distribution name>/<distribution version>
- The name and version must match respective names of Docker images.
-
-2. Component-specific data like build and run-time dependencies etc.
- These come from packaging/ subtree of particular component.
- E.g. data for "daemon" component are in subtree daemon/packaging/.
- The structure again has structure
- <distribution name>/<distribution version>.
- Files common for all distributions (like tests) are right in
- in packaging/ directory of given component.
-'''
-
-import argparse
-import logging
-from pathlib import Path
-import os
-import sys
-
-
-class TestEnv():
- '''
- Abstract way to schedule commands using different interpreters
-
- Reformat commands for different interpreters, e.g. Dockerfile, BASH, etc.
- '''
- def __init__(self, image):
- self.image = image
-
- def load_image(self):
- raise NotImplementedError()
-
- def run_cmds(self):
- raise NotImplementedError()
-
- def __str__(self):
- raise NotImplementedError()
-
-
-class DockerBuildEnv(TestEnv):
- '''
- Schedule commands as part of Docker build (Dockerfile)
- '''
- def __init__(self, image, srcdir):
- super().__init__(image)
- self.header = 'WORKDIR /root\nCOPY {} /root\n'.format(srcdir)
-
- def load_image(self):
- return 'FROM {0}:{1}\n'.format(self.image.name, self.image.version)
-
- def run_cmds(self):
- return '\n'.join('RUN {0}'.format(cmd) for cmd in self.image.cmds)
-
- def __str__(self):
- return self.load_image() + self.header + self.run_cmds()
-
-
-class Image():
- '''
- Abstraction to hide differences between distributions and their versions
- '''
- def __init__(self, img_path, name, version):
- self.img_path = img_path # scripts/distros/debian/9
- self.img_relpath = os.path.join(name, version) # debian/9
- self.name = name
- self.version = version
- self.actions = {}
- self.cmds = []
- self._init_cmds()
- # fill in Dockerfile with image preparation commands
-
- def _img_path(self, filename):
- '''Prepend distro-specific path before filename'''
- return os.path.join(self.img_path, filename)
-
- def _init_cmds(self):
- '''
- Read commands for image modification from
- '''
- for cmd in os.listdir(self.img_path):
- if cmd == 'prep': # multi-line commands are handled somewhere else
- continue
- with open(self._img_path(cmd)) as cmdfile:
- self.actions[cmd] = cmdfile.read().strip()
-
- def __str__(self):
- return '# image: {0}:{1}\n'.format(self.name, self.version) + '\n'.join(self.cmds)
-
- def action(self, action, arg):
- '''
- Schedule action with given argument, e.g. install package
-
- E.g. action "pkg_install" with argument "gcc" will schedule command
- read from image-specific file "distro/version/pkg_install" and append
- argument "arg". Result is like "apt-get install -y gcc".
- '''
- self.cmds.append('{0} {1}'.format(self.actions[action], arg))
-
- def action_arglist(self, action, cmpimgpath, filename):
- '''
- Plan single command with argumets equal to content of given text file
- '''
- try:
- with open(os.path.join(cmpimgpath, filename)) as listf:
- self.action(action, ' '.join(item.strip() for item in listf))
- except FileNotFoundError:
- pass
-
- def cmd(self, cmd):
- '''Schedule single command'''
- assert cmd
- self.cmds.append(cmd)
-
- def run_script(self, script):
- '''Shedule script from root directory'''
- if os.path.isfile(script):
- self.cmds.append(script)
-
- def img_script(self, script):
- '''Schedule script from image's directory'''
- path = self._img_path(script)
- assert os.path.isfile(path)
- self.run_script(path)
-
-
-class Component():
- '''
- API for single component of software (daemon etc.) independent on image
-
- comp_path must contain subtree <distribution name>/<distribution version>
- with files containing command specific for particular distribution
-
- image must be Image to work with
- '''
- def __init__(self, comp_path, image):
- self.comp_path = comp_path
- self.compimg_path = os.path.join(comp_path, image.img_relpath)
- self.image = image
- # Some components do not have external depedencies at the moment so
- # compimg_path may not exist. That is okay, we will just run their tests.
-
- def _comp_script(self, script):
- path = os.path.join(self.comp_path, script)
- if os.path.exists(path):
- self.image.cmd(path)
-
- def install_builddeps(self):
- self.image.run_script(self.compimg_path + '/pre-build.sh')
- self.image.action_arglist('pkg_install', self.compimg_path, 'builddeps')
-
- def build(self):
- self.image.run_script(self.compimg_path + '/build.sh')
-
- def install(self):
- self.image.run_script(self.compimg_path + '/install.sh')
-
- def remove_builddeps(self):
- self.image.action_arglist('pkg_remove', self.compimg_path, 'builddeps')
- self.image.run_script(self.compimg_path + '/post-build.sh')
-
- def install_rundeps(self):
- self.image.run_script(self.compimg_path + '/pre-run.sh')
- self.image.action_arglist('pkg_install', self.compimg_path, 'rundeps')
-
- def test(self):
- configcmdpath = os.path.join(self.comp_path, 'test.sh')
- configtestpath = os.path.join(self.comp_path, 'test.config')
- if os.path.exists(configcmdpath):
- self._comp_script('test.sh')
- elif os.path.exists(configtestpath):
- self.image.cmd('kresd -n -c {}'.format(configtestpath))
- self.image.run_script(self.compimg_path + '/post-run.sh')
-
-
-def foreach_component(components, action):
- '''Execute action for each component'''
- for comp in components:
- getattr(comp, action)()
-
-def main():
- logging.basicConfig(level=logging.DEBUG)
- argparser = argparse.ArgumentParser(
- formatter_class=argparse.RawTextHelpFormatter,
- description='''Generate Dockerfile to build/install/test given components.
-
-Examples:
-* Install build deps, build, install, remove build deps, and test kresd daemon:
- $ {n} debian 9 daemon/packaging > Dockerfile
-
-* Install build and run-time deps to prepare development image:
- $ find -name packaging | xargs {n} \\
- --build=false --install=false --remove-builddeps=false --test=false \\
- debian 9 > Dockerfile
-'''.format(n=sys.argv[0])
- )
- argparser.add_argument(
- '--builddeps', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--build', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--install', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--remove-builddeps', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--rundeps', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--test', default=True, type=bool, help='default: true')
- argparser.add_argument(
- '--srcdir', default=os.getcwd(), type=Path,
- help='directory to copy into new Docker image; default: .')
- argparser.add_argument(
- 'distro', help='name of distribution image, e.g. "debian"')
- argparser.add_argument(
- 'version', help='distribution version, e.g. 9')
- argparser.add_argument(
- 'components', nargs='+',
- help='one or more components to process; order is respected')
- args = argparser.parse_args()
-
- # all paths must be relative to toplevel Git dir
- if not os.path.exists('.luacheckrc') or not os.path.exists('NEWS'):
- sys.exit('This script must be executed from top of distribution tree!')
-
-
- # load images from disk
- imgpath = os.path.join('scripts/distros', args.distro, args.version)
- image = Image(imgpath, args.distro, args.version)
-
- components = [Component(comppath, image) for comppath in args.components]
- if args.builddeps:
- foreach_component(components, 'install_builddeps')
- if args.build:
- foreach_component(components, 'build')
- if args.install:
- foreach_component(components, 'install')
- if args.remove_builddeps:
- foreach_component(components, 'remove_builddeps')
- if args.rundeps:
- foreach_component(components, 'install_rundeps')
- if args.test:
- foreach_component(components, 'test')
- print(DockerBuildEnv(image, args.srcdir))
-
-if __name__ == '__main__':
- main()
+++ /dev/null
-#!/usr/bin/env python3
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import logging
-import os.path
-from pathlib import Path
-import subprocess
-import sys
-import tempfile
-import argparse
-
-
-DISTROS_PATH = Path(os.path.realpath('scripts/distros'))
-GEN_SCRIPT = Path(os.path.realpath('scripts/dockerfile_gen.py'))
-
-
-def unpack(archive, targetdir):
- '''
- Prepare workdir for Docker build by unpacking fresh distribution tarball
- '''
- logging.debug('unpacking fresh tarball %s into %s', archive, targetdir)
- subprocess.check_call(['tar', '-C', targetdir, '-xf', archive])
-
-
-def fresh_tarball():
- try:
- # make archive so we have clean state to test
- archive = subprocess.check_output('scripts/make-archive.sh')
- except subprocess.CalledProcessError as ex:
- logging.fatal('failed to generate fresh tarball: %s', ex.output)
- sys.exit(ex.returncode)
- return os.path.realpath(archive.strip())
-
-
-def get_distro_vers(distro_root):
- '''
- return list of (distro, version) pairs found in distro_root
- '''
- # transform list of paths like TOP/debian/9 into (debian, 9)
- dist_ver = [p.parts[-2:] for p
- in Path(distro_root).glob('*/*') if p.is_dir()]
- return list(dist_ver)
-
-
-def get_components(root):
- cmpl = [os.path.relpath(dirn, start=root) # relative names only
- for dirn, _, _ in os.walk(root)
- if (os.path.basename(dirn) == 'packaging' # path ends with
- and 'contrib' not in Path(dirn).parts)] # ignore contrib libs
- return list(cmpl)
-
-
-def test_combinations(distro_vers, components):
- tests = []
- for distro, ver in distro_vers:
- for comp in components:
- comps = ['scripts/distros', 'daemon/packaging'] # always include daemon
- if comp not in comps:
- comps.append(comp)
- tests.append([distro, ver, *comps])
- tests.sort()
- return tests
-
-
-def gen_dockerfile(args, tmpdir, srcdir):
- subprocess.check_call([GEN_SCRIPT,
- '--srcdir={}'.format(srcdir)] # dir in tar
- + args,
- stdout=open(tmpdir / 'Dockerfile', 'w'))
-
-
-def docker_build(tmpdir, delete):
- subprocess.check_call(
- ['docker',
- 'build',
- '--rm={}'.format(str(delete).lower()),
- '--network',
- 'host',
- tmpdir]
- )
-
-
-def find_test(required_tests, test_combination):
- '''
- Find test in test_combination in required_tests list
- '''
- for test in required_tests:
- if test_combination[len(test_combination)-1] == test[0]:
- return True
-
- return False
-
-
-def main():
- logging.basicConfig(level=logging.DEBUG)
-
- argparser = argparse.ArgumentParser(
- formatter_class=argparse.RawTextHelpFormatter,
- description='''Find all tests in current directory, generate Dockerfiles and run all Dockerfiles one by one.
-'''.format(n=sys.argv[0])
- )
- argparser.add_argument(
- '-t', '--test', action='append', nargs=1, help='Select one test to run')
- argparser.add_argument(
- '-l', '--list', action='store_true', help='Show all available tests')
-
- params = argparser.parse_args()
-
- distro_vers = get_distro_vers(DISTROS_PATH)
- components = get_components('.')
- logging.info('generating fresh tarball')
- archive = fresh_tarball()
- logging.debug('generated tarball %s', archive)
- # transform knot-resolver-1.5.0-70-gf1dbebdc.tar.xz -> knot-resolver-1.5.0-70-gf1dbebdc
- srcdir = os.path.basename(archive).decode('ascii').rsplit('.', maxsplit=2)[0]
- logging.debug('expected dir name in tarball: %s', srcdir)
-
- with tempfile.TemporaryDirectory() as tmpdir:
- tmpdir = Path(tmpdir)
- unpack(archive, tmpdir)
- baseimg = True # do not delete first image - it works as cache
- if params.list:
- print('Available tests: ')
- # all tests
- for args in test_combinations(distro_vers, components):
- if params.list:
- print('\t' + args[len(args)-1])
- continue
- if params.test:
- if not find_test(params.test, args):
- logging.debug('skip test for %s', args)
- continue
-
- logging.debug('generating Dockerfile for %s', args)
- gen_dockerfile(args, tmpdir, srcdir)
- from shutil import copyfile
- copyfile(os.path.join(tmpdir, "Dockerfile"),
- "/tmp/Dockerfile_" + args[0] + args[1] + "_" + args[-1].replace('/', '-'))
- continue
- logging.info('testing combination %s', args)
- docker_build(tmpdir, delete=not baseimg)
- baseimg = False
-
-
-if __name__ == '__main__':
- main()
projects / thirdparty / knot-resolver.git / commitdiff
