]> git.ipfire.org Git - thirdparty/knot-resolver.git/commitdiff
LXC-containers based CI with integration tests
authorVaclav Sraier <vaclav.sraier@nic.cz>
Mon, 24 May 2021 08:59:43 +0000 (08:59 +0000)
committerAleš Mrázek <ales.mrazek@nic.cz>
Fri, 8 Apr 2022 14:17:52 +0000 (16:17 +0200)
17 files changed:
manager/.gitignore
manager/.gitlab-ci.yml
manager/ci/README.md [deleted file]
manager/ci/devenv/Dockerfile [deleted file]
manager/containers/ci/Containerfile [new file with mode: 0644]
manager/containers/debian-supervisord/Containerfile
manager/containers/debian/Containerfile
manager/containers/dev/Containerfile
manager/integration/runner.py
manager/integration/tests/basic_crash/run
manager/integration/tests/basic_startup/run
manager/integration/tests/worker_count/run
manager/pyproject.toml
manager/scripts/_env.sh
manager/scripts/container-build [deleted file]
manager/scripts/container-run.py [deleted file]
manager/scripts/container.py [new file with mode: 0755]

index 294387c6bf51bb5f18754876e0dce7aeb8c0faf8..cfa0df52ca5079d1a64a00c0000f10ac4c276342 100644 (file)
@@ -10,3 +10,4 @@ dist/
 .tox/
 .vscode/
 /pkg
+.podman-cache/
\ No newline at end of file
index 0a07b4ed672ec0c8f8995883f7d4b5befc20a668..8ee7b9347a3d6b108474b0ec62058a61cbbdd54c 100644 (file)
@@ -2,23 +2,33 @@ stages:
   - image
   - check
 
-image: registry.nic.cz/knot/knot-resolver-manager/devenv:latest
-
+default:
+  image: registry.nic.cz/knot/knot-resolver-manager/knot-manager:ci
+  before_script:
+    # make sure Poetry is in $PATH
+    - source $HOME/.poetry/env
+    # there is already a pyproject.toml with installed dependencies in the root
+    # it has its own virtualenv and we want to use that env in a different directory
+    # so let's create a new one and replace it by the already existing one
+    # we don't care about destroying the environment in process, because it's going to be discarded anyway
+    - poetry env use $(which python3.6); ourpath=$(poetry env info -p); upperpath=$( (cd ..; poetry env info -p) ); rm -rf "$ourpath"; cp -a "$upperpath" "$ourpath"; mv /poetry.lock .
+    # the virtualenv we recycled might have beed slightly out of date. Let's quickly update it
+    - poetry install
+    # fix podman; see https://gitlab.nic.cz/labs/lxc-gitlab-runner#nesting-with-podman
+    - unset TMPDIR
+  tags:
+    - lxc
+    - amd64
 
 build:
-  image: docker:20-dind
   stage: image
-  tags:
-    - dind
-  variables:
-    IMAGE_TAG: $CI_REGISTRY/knot/knot-resolver-manager/devenv:latest
-  before_script:
-    - docker info
   script:
-    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
-    - docker build -t $IMAGE_TAG -f containers/dev/Containerfile .
-    - docker push $IMAGE_TAG
-
+    - poe container build --ci-login --push
+  only:
+    changes:
+      - pyproject.toml
+      - package.json
+      - containers/**/*
 
 lint:
   stage: check
@@ -29,3 +39,13 @@ test:
   stage: check
   script:
     - poe test
+    - poetry run coverage xml
+  artifacts:
+    reports:
+      cobertura: coverage.xml
+
+integration:
+  stage: check
+  script:
+    - poe container pull
+    - poe integration -n
diff --git a/manager/ci/README.md b/manager/ci/README.md
deleted file mode 100644 (file)
index c6f692d..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-Docker Build
-------------
-
-```
-$ docker build --no-cache -t registry.nic.cz/knot/knot-resolver-manager/devenv:latest devenv
-
-$ docker login registry.nic.cz
-$ docker push registry.nic.cz/knot/knot-resolver-manager/devenv:latest
-```
diff --git a/manager/ci/devenv/Dockerfile b/manager/ci/devenv/Dockerfile
deleted file mode 100644 (file)
index 92644b3..0000000
+++ /dev/null
@@ -1,56 +0,0 @@
-FROM docker.io/debian:latest
-
-ENV LC_ALL=C.UTF-8
-
-# pyenv setup deps
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y build-essential git ca-certificates
-
-# python build deps
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
-
-# Yarn and NodeJS (following https://github.com/nodesource/distributions/blob/master/README.md#debmanual)
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y gnupg lsb-release
-RUN curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
-ENV VERSION=node_14.x
-RUN echo "deb https://deb.nodesource.com/$VERSION $(lsb_release -s -c) main" | tee /etc/apt/sources.list.d/nodesource.list
-RUN echo "deb-src https://deb.nodesource.com/$VERSION $(lsb_release -s -c) main" | tee -a /etc/apt/sources.list.d/nodesource.list
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y nodejs
-RUN npm install -g yarn
-
-# Add non-root user
-RUN useradd -m -s /bin/bash user
-USER user
-WORKDIR /home/user
-
-# install pyenv
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-ENV PATH="/home/user/.pyenv/bin:$PATH"
-# These would should be run in the shell that will be running pyenv. But it can hopefully work without it
-# RUN eval "$(pyenv init -)"
-# RUN eval "$(pyenv virtualenv-init -)"
-
-# install all required versions of python via pyenv
-RUN pyenv install 3.6.12
-RUN pyenv install 3.7.9
-RUN pyenv install 3.8.7
-RUN pyenv install 3.9.1
-
-# install poetry
-USER root
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y python3 python3-distutils-extra
-USER user
-
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 -
-ENV PATH="/home/user/.poetry/bin:$PATH"
-# force Poetry to run under python3
-RUN sed -i 's/env python/env python3/' .poetry/bin/poetry
-# force Poetry to use local .venv/ directory that we can cache
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-
-
-# install additional project dependencies
-USER root
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y libcairo2-dev libglib2.0-0 libgirepository1.0-dev
-RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y systemd
-USER user
-
diff --git a/manager/containers/ci/Containerfile b/manager/containers/ci/Containerfile
new file mode 100644 (file)
index 0000000..5de016b
--- /dev/null
@@ -0,0 +1,48 @@
+FROM registry.nic.cz/labs/lxc-gitlab-runner/fedora-34:podman
+
+# Install Python and NodeJS
+RUN dnf install -y python3.6 nodejs python3-gobject pkg-config cairo-devel gcc python3-devel gobject-introspection-devel cairo-gobject-devel which \
+  && dnf clean all
+
+# Install Poetry
+RUN python3 -m pip install -U pip \
+  && curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3 - \
+  && source $HOME/.poetry/env \
+  # not exactly required, but helpful
+  && python3 -m pip install poethepoet
+
+# force Poetry to use local .venv/ directory that we can cache
+ENV POETRY_NO_INTERACTION=1 \
+  # python:
+  PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PYTHONDONTWRITEBYTECODE=1
+
+
+# How does this work?
+# ===================
+#
+# NPM dependencies are installed globally. There is no problem with that.
+#
+# Python dependencies are however installed into a virtualenv created by Poetry. Why you might ask?
+# Because we can't change the default python interpreter without virtualenv. This creates a problem,
+# that the virtualenv is created for a directory different than the one, where CI will run.
+#
+# Yup, that's a slight issue, that has to be fixed before running anything. This migration step is however
+# quick. It's just copying files locally and there's not a ton of them. This virtualenv migration step is
+# therefore done every time a CI job starts.
+#
+# How does it speed up CI?
+# ========================
+#
+# We do not have to install the dependencies every single time. They are cached in the container itself and 
+# we can rebuild it only when it's definition or the list of dependencies changes.
+
+COPY pyproject.toml package.json .
+RUN source $HOME/.poetry/env \
+  && poetry config --list \
+  && poetry env use $(which python3.6) \
+  && poetry env info \
+  && poetry install --no-interaction --no-ansi \
+  && npm install -g $(python -c "import json; print(*(k for k in json.loads(open('package.json').read())['dependencies']))")
\ No newline at end of file
index 5c11746da3213b5ccaa0ebc8070e11e998ba44a0..9a4b5fe0ca31fe2e92dd86381f2cf62cbebf5b05 100644 (file)
@@ -1,4 +1,4 @@
-FROM knot-manager:debian
+FROM registry.nic.cz/knot/knot-resolver-manager/knot-manager:debian
 
 # Remove systemd
 # RUN apt-get remove -y systemd
index c8621efb84c6c8c4cdc034dcab90665ee698a4c0..9445f14b9c580e30b5f7d0c6db5daab9bbf2db72 100644 (file)
@@ -21,7 +21,6 @@ ENV \
   NODE_VERSION=node_14.x
 ENV LC_ALL=C.UTF-8
 
-# System deps:
 # System deps:
 RUN apt-get update \
   && apt-get install --no-install-recommends -y \
@@ -62,7 +61,7 @@ COPY ./config/kres-manager.yaml /etc/knot-resolver
 
 # Copy only requirements, to cache them in docker layer
 # no poetry.lock, because here we have a different python version
-COPY ./pyproject.toml ./yarn.lock ./package.json /code/
+COPY ./pyproject.toml /code/
 
 WORKDIR /code
 
index 9bbd9b8dfbf11aae10a6fc2227ee162f8506e41d..a5fc6e0263567a55cd206009c994228635806364 100644 (file)
@@ -47,7 +47,7 @@ RUN apt-get update \
   && dpkg -i knot-resolver-release.deb \
   && rm knot-resolver-release.deb \
   && apt-get update && apt-get install -y --no-install-recommends knot-resolver \
-  # Installing Yarn and NodeJS
+  # Installing NodeJS
   && curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
   && echo "deb https://deb.nodesource.com/$NODE_VERSION $(lsb_release -s -c) main" | tee /etc/apt/sources.list.d/nodesource.list \
   && echo "deb-src https://deb.nodesource.com/$NODE_VERSION $(lsb_release -s -c) main" | tee -a /etc/apt/sources.list.d/nodesource.list \
index 3a52bf24e4934fa6662ee5154098d148614a47cb..0a5cd20bf0830e8c9ebee07f8a8ad274d6aabab6 100644 (file)
@@ -2,7 +2,7 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import Dict, List, TypeVar
+from typing import Dict, List, NoReturn, TypeVar
 
 import click
 import toml
@@ -51,11 +51,12 @@ class Test:
         self._images = [ str(img) for img in config["images"]]
 
     
-    def run(self, inspect_failed: bool =False):
+    def run(self, inspect_failed: bool =False) -> bool:
+        success = True
         for image in self._images:
             print(f"Running test {Colors.YELLOW}{self.name}{Colors.RESET} within container {Colors.YELLOW}{image}{Colors.RESET}")
             print(f"----------------------------{Colors.BRIGHT_BLACK}")
-            cmd: List[str] = ["../scripts/container-run.py"] + (["-i"] if inspect_failed else []) + flatten([["-m", f"{k}:{v}"] for k,v in self._mounts.items()]) + [image] + self._cmd
+            cmd: List[str] = ["../scripts/container.py", "run"] + (["-i"] if inspect_failed else []) + flatten([["-m", f"{k}:{v}"] for k,v in self._mounts.items()]) + [image] + self._cmd
 
             # run and relay output
             exit_code = subprocess.call(cmd)
@@ -66,6 +67,8 @@ class Test:
                 print(
                     f"{Colors.RED}Test failed with exit code {exit_code}{Colors.RESET}"
                 )
+            success = success and exit_code == 0
+        return success
 
 class TestRunner:
     _TEST_DIRECTORY = "tests"
@@ -99,7 +102,7 @@ class TestRunner:
         default=False,
         is_flag=True,
     )
-    def run(tests: List[str] = [], inspect_failed: bool = False, no_build: bool = False):
+    def run(tests: List[str] = [], inspect_failed: bool = False, no_build: bool = False) -> NoReturn:
         """Run TESTS
 
         If no TESTS are specified, runs them all.
@@ -107,10 +110,11 @@ class TestRunner:
 
         # build all test containers
         if not no_build:
-            ret = subprocess.call("poe container-build", shell=True)
+            ret = subprocess.call("poe container build", shell=True)
             assert ret == 0
 
         # Run the tests
+        success = True
         for test_path in TestRunner._list_tests():
             test = Test(test_path)
 
@@ -118,7 +122,13 @@ class TestRunner:
                 print(f"Skipping test {Colors.YELLOW}{test.name}{Colors.RESET}")
                 continue
 
-            test.run(inspect_failed)
+            res = test.run(inspect_failed)
+            success = success and res
+        
+        if not success:
+            sys.exit(1)
+        else:
+            sys.exit(0)
 
 
 if __name__ == "__main__":
index 4634a6f6e4c933b8bfdb4b32e29ded7672a1bb29..4e48e036bf8ef7c9d39a110a781a180d74e1a8eb 100755 (executable)
@@ -8,17 +8,18 @@ echo "Starting manager..."
 systemctl start knot-resolver-manager.service
 
 # give it time to start
-sleep 1
+while [ ! -e /tmp/manager.sock ]; do sleep 0.5; done
 
 # start kresd instances and verify, that it works
 python3 run_test.py
 
 # kill the manager and start it again
 systemctl kill --signal=SIGKILL knot-resolver-manager.service
+rm /tmp/manager.sock
 systemctl start knot-resolver-manager.service
 
 # wait for proper startup
-sleep 1.5
+while [ ! -e /tmp/manager.sock ]; do sleep 0.5; done
 
 # run the same test again testing, that instances can start and that the count is correct
 # because there should be kresd instances left after the first run, this tests that the initialization of the manager properly finds them
index 2b0a1bfdb8e21f4c086305faeb276c31c588008f..8162acae334445a0f2a219c5162c4730d2808564 100755 (executable)
@@ -9,7 +9,7 @@ cmd=$(grep ExecStart /etc/systemd/system/knot-resolver-manager.service | sed 's/
 bash -c "cd /code; $cmd" &
 
 # give it time to start
-sleep 2
+while [ ! -e /tmp/manager.sock ]; do sleep 0.5; done
 
 python3 send_request.py
 
index 9d946f2e30ddae744002b1f5bf71c6ed29215552..11ed0102f4293ba41abaca647bbcc3af91aeda7c 100755 (executable)
@@ -8,6 +8,6 @@ echo "Starting manager..."
 systemctl start knot-resolver-manager.service
 
 # give it time to start
-sleep 1
+while [ ! -e /tmp/manager.sock ]; do sleep 0.5; done
 
 python3 run_test.py
index 63aceecabfd75c15c603a8dcd98d05196c68672e..b41e5c2de194a6b06028fd1d73e6c52423de58a6 100644 (file)
@@ -41,8 +41,7 @@ check = { cmd = "scripts/codecheck", help = "Run static code analysis" }
 format = { shell = "poetry run black knot_resolver_manager/ tests/; isort -rc .", help = "Run code formatter" }
 fixdeps = { shell = "poetry install; npm install", help = "Install/update dependencies according to configuration files"}
 commit = { shell = "scripts/commit", help = "Invoke every single check before commiting" }
-container-build = { cmd = "scripts/container-build", help = "Build containers (no arguments = all, otherwise arguments are tags that should be built)" }
-container-run = { cmd = "scripts/container-run.py", help = "Run a container" }
+container = { cmd = "scripts/container.py", help = "Manage containers" }
 clean = """
   rm -rf .coverage
          .mypy_cache
index f5ec18abbbf956cc6b796384e1db407391e596f3..ca80531c511361af294369391f562e757b85a1b7 100644 (file)
@@ -17,7 +17,7 @@ fi
 cd $gitroot
 
 # ensure consistent environment with virtualenv
-if test -z "$VIRTUAL_ENV" -a "$CI" != "true"; then
+if test -z "$VIRTUAL_ENV" -a "$CI" != "true" -a -z "$KNOT_ENV"; then
        echo -e "${yellow}You are NOT running the script within the project's virtual environment.${reset}"
        echo -e "Do you want to continue regardless? [yN]"
        read cont
diff --git a/manager/scripts/container-build b/manager/scripts/container-build
deleted file mode 100755 (executable)
index 032b248..0000000
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# ensure consistent behaviour
-src_dir="$(dirname "$(realpath "$0")")"
-source $src_dir/_env.sh
-
-
-if test "$#" -eq 0; then
-    containers="$(find containers -maxdepth 1 -type d -printf '%f\n' | grep -v containers)"
-else
-    containers="$@"
-fi
-
-
-# build all configured containers
-for tag in $containers; do
-    echo -e "Building ${yellow}knot-manager:${tag}${reset}${bright_black}"
-    podman build -t "knot-manager:$tag" -f "containers/$tag/Containerfile" .
-    echo -e "${reset}Build finished"
-done
\ No newline at end of file
diff --git a/manager/scripts/container-run.py b/manager/scripts/container-run.py
deleted file mode 100755 (executable)
index 0e567ee..0000000
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-import sys
-import time
-from pathlib import Path
-from typing import Dict, List, NoReturn, Optional
-
-import click
-
-PODMAN_EXECUTABLE = "/usr/bin/podman"
-
-
-def start_detached(
-    image: str, publish: List[int] = [], ro_mounts: Dict[Path, Path] = {}
-) -> str:
-    """Start a detached container"""
-    options = [f"--publish={port}:{port}/tcp" for port in publish] + [
-        f"--mount=type=bind,source={str(src)},destination={str(dst)},ro=true"
-        for src, dst in ro_mounts.items()
-    ]
-    command = ["podman", "run", "--rm", "-d", *options, image]
-    proc = subprocess.run(
-        command, shell=False, executable=PODMAN_EXECUTABLE, stdout=subprocess.PIPE
-    )
-    assert proc.returncode == 0
-    return str(proc.stdout, "utf8").strip()
-
-
-def exec(container_id: str, cmd: List[str]) -> int:
-    command = ["podman", "exec", container_id] + cmd
-    return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
-
-
-def exec_interactive(container_id: str, cmd: List[str]) -> int:
-    command = ["podman", "exec", "-ti", container_id] + cmd
-    return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
-
-
-def stop(container_id: str):
-    command = ["podman", "stop", container_id]
-    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
-    assert ret == 0
-
-
-def _get_git_root() -> Path:
-    result = subprocess.run(
-        "git rev-parse --show-toplevel", shell=True, stdout=subprocess.PIPE
-    )
-    return Path(str(result.stdout, encoding="utf8").strip())
-
-
-@click.command()
-@click.argument("image", nargs=1)
-@click.argument("command", nargs=-1)
-@click.option(
-    "-p", "--publish", "publish", multiple=True, type=int, help="Port which should be published"
-)
-@click.option(
-    "-m",
-    "--mount",
-    "mount",
-    multiple=True,
-    nargs=1,
-    type=str,
-    help="Read-only bind mounts into the container, value /path/on/host:/path/in/container",
-)
-@click.option(
-    "-c",
-    "--code",
-    "mount_code",
-    default=False,
-    is_flag=True,
-    type=bool,
-    help="Shortcut to mount gitroot into /code",
-)
-@click.option(
-    "-i",
-    "--interactive",
-    "interactive_inspection",
-    default=False,
-    is_flag=True,
-    type=bool,
-    help="Drop into interactive shell if the command fails"
-)
-def main(
-    image: str,
-    command: List[str],
-    publish: Optional[List[int]],
-    mount: Optional[List[str]],
-    mount_code: bool,
-    interactive_inspection: bool,
-) -> NoReturn:
-    # make sure arguments have the correct type
-    image = str(image)
-    command = list(command)
-    publishI = [] if publish is None else [int(p) for p in publish]
-    mountI = [] if mount is None else [x.split(":") for x in mount]
-    mount_path = {Path(x[0]).absolute(): Path(x[1]).absolute() for x in mountI}
-    for src_path in mount_path:
-        if not src_path.exists():
-            print(
-                f'The specified path "{str(src_path)}" does not exist on the host system',
-                file=sys.stderr,
-            )
-            exit(1)
-    if mount_code:
-        mount_path[_get_git_root()] = Path("/code")
-
-    cont = start_detached(image, publish=publishI, ro_mounts=mount_path)
-    # wait for the container to boot properly
-    time.sleep(0.5)
-    # run the command
-    exit_code = exec_interactive(cont, command)
-
-    if interactive_inspection and exit_code != 0:
-        print(f"The command {command} failed with exit code {exit_code}.")
-        print("Dropping into an interactive shell as requested. Stop the shell to stop the whole container.")
-        print("-----------------------------")
-        exec_interactive(cont, ["/bin/bash"])
-
-    # stop the container
-    stop(cont)
-    sys.exit(exit_code)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/manager/scripts/container.py b/manager/scripts/container.py
new file mode 100755 (executable)
index 0000000..15f7f57
--- /dev/null
@@ -0,0 +1,268 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Dict, List, NoReturn, Optional
+from os import environ
+import atexit
+
+import click
+
+def _get_git_root() -> Path:
+    result = subprocess.run(
+        "git rev-parse --show-toplevel", shell=True, stdout=subprocess.PIPE
+    )
+    return Path(str(result.stdout, encoding="utf8").strip())
+
+
+GIT_ROOT: Path = _get_git_root()
+PODMAN_EXECUTABLE = "/usr/bin/podman"
+CACHE_DIR: Path = GIT_ROOT / ".podman-cache"
+
+
+
+
+def _start_detached(
+    image: str, publish: List[int] = [], ro_mounts: Dict[Path, Path] = {}
+) -> str:
+    """Start a detached container"""
+    options = [f"--publish={port}:{port}/tcp" for port in publish] + [
+        f"--volume={str(src)}:{str(dst)}:O"
+        for src, dst in ro_mounts.items()
+    ]
+    command = ["podman", "run", "--rm", "-d", *options, image]
+    proc = subprocess.run(
+        command, shell=False, executable=PODMAN_EXECUTABLE, stdout=subprocess.PIPE
+    )
+    assert proc.returncode == 0
+    return str(proc.stdout, "utf8").strip()
+
+
+def _exec(container_id: str, cmd: List[str]) -> int:
+    command = ["podman", "exec", container_id] + cmd
+    return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+
+
+def _exec_interactive(container_id: str, cmd: List[str]) -> int:
+    command = ["podman", "exec", "-ti", container_id] + cmd
+    return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+
+
+def _stop(container_id: str):
+    command = ["podman", "stop", container_id]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+
+def _list_available_image_tags() -> List[str]:
+    res: List[str] = []
+    for c in (GIT_ROOT / "containers").iterdir():
+        if c.is_dir():
+            res.append(c.name)
+    res.sort()  # make the order reproducible
+    return res
+
+
+def _extract_tag_from_name(name: str, all: List[str] = _list_available_image_tags()) -> str:
+    if ":" in name:
+        s = name.split(":")
+        if not s[0].endswith("knot-manager"):
+            click.secho(f"Unexpected image name \'{s[0]}\', expected \'knot-manager\'", fg="red")
+            sys.exit(1)
+        name = s[-1]
+    
+    if not name in all:
+        click.secho(f"Unexpected tag \'{name}\'", fg="red")
+        click.secho(f"Available tags are [{' '.join(all)}]", fg="yellow")
+        sys.exit(1)
+    
+    return name
+
+def _get_tags_to_work_on(args: List[str]) -> List[str]:
+    args = list(args)
+
+    all = _list_available_image_tags()
+
+    # convert to tags, if the user specified full names
+    for i,a in enumerate(args):
+        args[i] = _extract_tag_from_name(a, all)
+
+    if len(args) == 0:
+        args = all
+
+    return args
+
+
+def _full_name_from_tag(tag: str) -> str:
+    return f"registry.nic.cz/knot/knot-resolver-manager/knot-manager:{tag}"
+
+
+def _build(tag: str):
+    command = ["podman", "build", "-f", str(GIT_ROOT / "containers" / tag / "Containerfile"), "-t", _full_name_from_tag(tag), str(GIT_ROOT)]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+
+def _pull(tag: str):
+    command = ["podman", "pull", _full_name_from_tag(tag)]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+
+def _push(tag: str):
+    command = ["podman", "push", _full_name_from_tag(tag)]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+
+def _login_ci():
+    command = ["podman", "login", "-u", environ["CI_REGISTRY_USER"], "-p", environ["CI_REGISTRY_PASSWORD"], environ["CI_REGISTRY"]]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+
+def _save(tag: str):
+    CACHE_DIR.mkdir(exist_ok=True)
+    command = ["podman", "save", "--format", "oci-archive", "-o", str(CACHE_DIR / (tag + ".tar")), _full_name_from_tag(tag)]
+    ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+    assert ret == 0
+
+def _load(tag: str):
+    cache_file = CACHE_DIR / (tag + ".tar")
+    if cache_file.exists():
+        command = ["podman", "load", "-i", str(CACHE_DIR / (tag + ".tar"))]
+        ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
+        assert ret == 0
+
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command(help="Pull CI built images")
+@click.argument("images", nargs=-1)
+def pull(images: List[str]):
+    tags = _get_tags_to_work_on(images)
+
+    for tag in tags:
+        click.secho(f"Pulling image with tag {tag}", fg="yellow")
+        _pull(tag)
+
+
+
+@main.command(help="Build project containers")
+@click.argument("images", nargs=-1)
+@click.option("-f", "--fetch", "fetch", is_flag=True, default=False, type=bool, help="Pull before building")
+@click.option("--ci-login", "ci_login", is_flag=True, default=False, type=bool, help="Login to registry in CI")
+@click.option("-p", "--push", "push", is_flag=True, default=False, type=bool, help="Push images after building")
+@click.option("--file-cache", is_flag=True, default=False, help="Try to utilise file cache")
+def build(images: List[str], fetch: bool, ci_login: bool, push: bool, file_cache: bool):
+    tags = _get_tags_to_work_on(images)
+
+    if ci_login:
+        _login_ci()
+    
+    for tag in tags:
+        if fetch:
+            click.secho(f"Pulling image with tag {tag}", fg="yellow")
+            _pull(tag)
+        
+        if file_cache:
+            _load(tag)
+
+        click.secho(f"Building image with tag {tag}", fg="yellow")
+        _build(tag)
+
+        if push:
+            click.secho(f"Pushing image with {tag}", fg="yellow")
+            _push(tag)
+        
+        if file_cache:
+            _save(tag)
+
+
+@main.command(help="Run project containers")
+@click.argument("image", nargs=1)
+@click.argument("command", nargs=-1)
+@click.option(
+    "-p", "--publish", "publish", multiple=True, type=int, help="Port which should be published"
+)
+@click.option(
+    "-m",
+    "--mount",
+    "mount",
+    multiple=True,
+    nargs=1,
+    type=str,
+    help="Read-only bind mounts into the container, value /path/on/host:/path/in/container",
+)
+@click.option(
+    "-c",
+    "--code",
+    "mount_code",
+    default=False,
+    is_flag=True,
+    type=bool,
+    help="Shortcut to mount gitroot into /code",
+)
+@click.option(
+    "-i",
+    "--interactive",
+    "interactive_inspection",
+    default=False,
+    is_flag=True,
+    type=bool,
+    help="Drop into interactive shell if the command fails"
+)
+def run(
+    image: str,
+    command: List[str],
+    publish: Optional[List[int]],
+    mount: Optional[List[str]],
+    mount_code: bool,
+    interactive_inspection: bool,
+) -> NoReturn:
+    # make sure arguments have the correct type
+    tag = _extract_tag_from_name(image)
+    command = list(command)
+    publishI = [] if publish is None else [int(p) for p in publish]
+    mountI = [] if mount is None else [x.split(":") for x in mount]
+    mount_path = {Path(x[0]).absolute(): Path(x[1]).absolute() for x in mountI}
+    for src_path in mount_path:
+        if not src_path.exists():
+            print(
+                f'The specified path "{str(src_path)}" does not exist on the host system',
+                file=sys.stderr,
+            )
+            exit(1)
+    if mount_code:
+        mount_path[GIT_ROOT] = Path("/code")
+
+    cont = _start_detached(_full_name_from_tag(tag), publish=publishI, ro_mounts=mount_path)
+
+    # register cleanup function
+    def cleanup():
+        _stop(cont)
+    atexit.register(cleanup)
+
+    # wait for the container to boot properly
+    time.sleep(0.5)
+    # run the command
+    exit_code = _exec_interactive(cont, command)
+
+    if interactive_inspection and exit_code != 0:
+        print(f"The command {command} failed with exit code {exit_code}.")
+        print("Dropping into an interactive shell as requested. Stop the shell to stop the whole container.")
+        print("-----------------------------")
+        _exec_interactive(cont, ["/bin/bash"])
+
+    # the container should be stopped by the `atexit` module
+    sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file