From: Vasek Sraier Date: Fri, 12 Mar 2021 14:13:46 +0000 (+0100) Subject: bug fixes in order to make the basic startup integration test pass X-Git-Tag: v6.0.0a1~220 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fcf6d7df22cba57abb9d52bd66c168a5c88b9a8a;p=thirdparty%2Fknot-resolver.git bug fixes in order to make the basic startup integration test pass --- diff --git a/manager/ci/devenv/Dockerfile b/manager/ci/devenv/Dockerfile index 4d80556fe..92644b381 100644 --- a/manager/ci/devenv/Dockerfile +++ b/manager/ci/devenv/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:latest +FROM docker.io/debian:latest ENV LC_ALL=C.UTF-8 diff --git a/manager/integration/runner.py b/manager/integration/runner.py index 87020206e..314b16289 100644 --- a/manager/integration/runner.py +++ b/manager/integration/runner.py @@ -9,6 +9,8 @@ import time import sys import requests import hashlib +import click +import json from _hashlib import HASH as Hash from pathlib import Path, PurePath @@ -99,10 +101,13 @@ class PodmanServiceManager: def _api_build_container(self, image_name: str, data: BinaryIO): response = requests.post( - self._create_url("libpod/build"), params=[("t", image_name)], data=data + self._create_url("libpod/build"), params=[("t", image_name)], data=data, stream=True ) response.raise_for_status() + for line in response.iter_lines(): + print(f"\t\t{json.loads(str(line, 'utf8'))['stream'].rstrip()}") + def _read_and_remove_hashfile(self, context_dir: Path) -> Optional[str]: hashfile: Path = context_dir / PodmanServiceManager._HASHFILE_NAME if hashfile.exists(): @@ -190,10 +195,13 @@ class PodmanServiceManager: def _api_start_exec(self, exec_id): response = requests.post( - self._create_url(f"libpod/exec/{exec_id}/start"), json={} + self._create_url(f"libpod/exec/{exec_id}/start"), json={}, stream=True ) response.raise_for_status() + for line in response.iter_lines(): + print(f"\t\t{str(line, 'utf8').rstrip()}") + def _api_get_exec_exit_code(self, exec_id) -> int: response = requests.get(self._create_url(f"libpod/exec/{exec_id}/json")) response.raise_for_status() @@ -208,7 +216,7 @@ class PodmanServiceManager: response.raise_for_status() def start_temporary_and_wait( - self, image: str, command: List[str], bind_mount_ro: Dict[PurePath, PurePath] = {} + self, image: str, command: List[str], bind_mount_ro: Dict[PurePath, PurePath] = {}, wait_interactively: bool = False ) -> int: # start the container container_id = self._api_create_container(image, bind_mount_ro) @@ -222,6 +230,11 @@ class PodmanServiceManager: self._api_start_exec(exec_id) test_exit_code = self._api_get_exec_exit_code(exec_id) + if wait_interactively: + print(f"\tExit code is {test_exit_code}") + print("\tPress [ENTER] to continue...") + input() + # issue shutdown command to the container exec_id = self._api_create_exec(container_id, ["systemctl", "poweroff"]) self._api_start_exec(exec_id) @@ -240,7 +253,7 @@ class Colors: def _get_git_root() -> PurePath: - result = subprocess.run("git rev-parse --show-toplevel", shell=True, capture_output=True) + result = subprocess.run("git rev-parse --show-toplevel", shell=True, stdout=subprocess.PIPE) return PurePath(str(result.stdout, encoding='utf8').strip()) class TestRunner: @@ -259,10 +272,18 @@ class TestRunner: ] @staticmethod - def run(): + @click.command() + @click.argument('tests', nargs=-1) + @click.option("-w", "--wait", help="Do not stop the test container immediately, wait for confirmation.", default=False, is_flag=True) + def run(tests: List[str] = [], wait: bool = False): with PodmanService() as manager: for test_path in TestRunner._list_tests(): test_name = test_path.absolute().name + + if len(tests) != 0 and test_name not in tests: + print(f"Skipping test {Colors.YELLOW}{test_name}{Colors.RESET}") + continue + print(f"Running test {Colors.YELLOW}{test_name}{Colors.RESET}") image = "knot_test_" + test_name print("\tBuilding...") @@ -274,7 +295,8 @@ class TestRunner: bind_mount_ro={ _get_git_root(): PurePath('/repo'), test_path.absolute(): '/test' - } + }, + wait_interactively=wait ) if exit_code == 0: print(f"\t{Colors.GREEN}Test succeeded{Colors.RESET}") diff --git a/manager/integration/tests/basic_startup/Dockerfile b/manager/integration/tests/basic_startup/Dockerfile index 9c27a9cce..c6efb7063 100644 --- a/manager/integration/tests/basic_startup/Dockerfile +++ b/manager/integration/tests/basic_startup/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:latest +FROM docker.io/debian:latest ENV LC_ALL=C.UTF-8 @@ -33,5 +33,6 @@ RUN apt-get update && apt-get install --no-install-recommends --no-install-sugge # install test dependencies RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl +RUN pip3 install requests requests-unixsocket CMD ["/bin/systemd"] diff --git a/manager/integration/tests/basic_startup/run b/manager/integration/tests/basic_startup/run index d25cbbf6e..959ccced9 100755 --- a/manager/integration/tests/basic_startup/run +++ b/manager/integration/tests/basic_startup/run @@ -5,12 +5,14 @@ set -e cd /test echo "Starting manager..." -cp knot-manager.service /etc/systemd/system +cp knot-resolver.service /etc/systemd/system systemctl daemon-reload -systemctl start knot-manager.service +systemctl start knot-resolver.service +# give it time to start +sleep 1 -echo -e "lua_config: \"\"\nnum_workers: 4" | curl -X POST --unix-socket /tmp/manager.sock localhost/config -f --data-binary @- +python3 send_request.py # assert that any kresd process is running systemctl status | grep kresd diff --git a/manager/integration/tests/basic_startup/send_request.py b/manager/integration/tests/basic_startup/send_request.py new file mode 100644 index 000000000..eb9b31858 --- /dev/null +++ b/manager/integration/tests/basic_startup/send_request.py @@ -0,0 +1,43 @@ + + + +import requests +import requests_unixsocket + +# patch requests library so that it supports unix socket +requests_unixsocket.monkeypatch() + +# prepare the payload +LUA_CONFIG = """ +-- SPDX-License-Identifier: CC0-1.0 +-- vim:syntax=lua:set ts=4 sw=4: +-- Refer to manual: https://knot-resolver.readthedocs.org/en/stable/ + +-- Network interface configuration +net.listen('127.0.0.1', 53, { kind = 'dns' }) +net.listen('127.0.0.1', 853, { kind = 'tls' }) +--net.listen('127.0.0.1', 443, { kind = 'doh2' }) +net.listen('::1', 53, { kind = 'dns', freebind = true }) +net.listen('::1', 853, { kind = 'tls', freebind = true }) +--net.listen('::1', 443, { kind = 'doh2' }) + +-- Load useful modules +modules = { + 'hints > iterate', -- Load /etc/hosts and allow custom root hints + 'stats', -- Track internal statistics + 'predict', -- Prefetch expiring/frequent records +} + +-- Cache size +cache.size = 100 * MB +""" +PREPROCESSED_CONFIG = "\n ".join(LUA_CONFIG.splitlines(keepends=False)) +PAYLOAD = f""" +num_workers: 4 +lua_config: | +{ PREPROCESSED_CONFIG } +""" + +# send the config +r = requests.post('http+unix://%2Ftmp%2Fmanager.sock/config', data=PAYLOAD) +r.raise_for_status() \ No newline at end of file diff --git a/manager/integration/tests/dummy/Dockerfile b/manager/integration/tests/dummy/Dockerfile index 05b007838..c811f39fc 100644 --- a/manager/integration/tests/dummy/Dockerfile +++ b/manager/integration/tests/dummy/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM docker.io/debian:buster RUN apt-get update && apt-get install -y systemd sudo diff --git a/manager/knot_resolver_manager/kresd_manager.py b/manager/knot_resolver_manager/kresd_manager.py index 85acaa7b9..803f1b204 100644 --- a/manager/knot_resolver_manager/kresd_manager.py +++ b/manager/knot_resolver_manager/kresd_manager.py @@ -77,7 +77,9 @@ class KresdManager: await self._spawn_new_child() async def _write_config(self, config: YAML): - raise NotImplementedError() + # FIXME: this code is blocking!!! + with open("/etc/knot-resolver/kresd.conf", 'w') as f: + f.write(config['lua_config'].text) async def apply_config(self, config: YAML): async with self._children_lock: diff --git a/manager/poetry.lock b/manager/poetry.lock index 2ea8f77f1..46149567e 100644 --- a/manager/poetry.lock +++ b/manager/poetry.lock @@ -676,6 +676,18 @@ urllib3 = ">=1.21.1,<1.27" security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +[[package]] +name = "requests-unixsocket" +version = "0.2.0" +description = "Use requests to talk HTTP via a UNIX domain socket" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +requests = ">=1.1" +urllib3 = ">=1.8" + [[package]] name = "requirements-detector" version = "0.7" @@ -912,7 +924,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6.12" -content-hash = "3c88907317606698cea3e6abbf6e5d84a68a32f942f90e3115f8a02f88539051" +content-hash = "19afffd5ea930c525a9a723ede432e8fcffcfef19801c15e721d83d6f79cdeae" [metadata.files] aiohttp = [ @@ -1303,18 +1315,26 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, @@ -1366,6 +1386,10 @@ requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] +requests-unixsocket = [ + {file = "requests-unixsocket-0.2.0.tar.gz", hash = "sha256:9e5c1a20afc3cf786197ae59c79bcdb0e7565f218f27df5f891307ee8817c1ea"}, + {file = "requests_unixsocket-0.2.0-py2.py3-none-any.whl", hash = "sha256:014d07bfb66dc805a011a8b4b306cf4ec96d2eddb589f6b2b5765e626f0dc0cc"}, +] requirements-detector = [ {file = "requirements-detector-0.7.tar.gz", hash = "sha256:0d1e13e61ed243f9c3c86e6cbb19980bcb3a0e0619cde2ec1f3af70fdbee6f7b"}, ] diff --git a/manager/pyproject.toml b/manager/pyproject.toml index dab0a5dd1..23e6a0d3a 100644 --- a/manager/pyproject.toml +++ b/manager/pyproject.toml @@ -23,6 +23,8 @@ tox-pyenv = "^1.1.0" poethepoet = "^0.9.0" prospector = {extras = ["with_mypy", "with_bandit"], version = "^1.3.1"} requests = "^2.25.1" +requests-unixsocket = "^0.2.0" +click = "^7.1.2" [tool.poe.tasks] run = { cmd = "python -m knot_resolver_manager", help = "Run the manager" }