From 7500b4e8dbad79a32ad3645bc275ae3f31316883 Mon Sep 17 00:00:00 2001 From: Vasek Sraier Date: Wed, 24 Mar 2021 17:15:13 +0100 Subject: [PATCH] debugging: created run-debug poe command which starts a container with debugpy server --- .../knot-resolver-manager.service} | 2 +- manager/containers/README.md | 8 ++- manager/containers/debian/Containerfile | 3 + manager/containers/dev/Containerfile | 3 + manager/integration/runner.py | 2 +- manager/integration/tests/basic_startup/run | 4 +- .../tests/worker_count/knot-resolver.service | 12 ---- manager/integration/tests/worker_count/run | 4 +- manager/poetry.lock | 66 ++++++++++++++++++- manager/pyproject.toml | 5 +- .../{build-containers => container-buildall} | 0 manager/scripts/container-run.py | 52 +++++++++++++++ manager/scripts/run-debug | 18 +++++ 13 files changed, 156 insertions(+), 23 deletions(-) rename manager/{integration/tests/basic_startup/knot-resolver.service => config/knot-resolver-manager.service} (87%) delete mode 100644 manager/integration/tests/worker_count/knot-resolver.service rename manager/scripts/{build-containers => container-buildall} (100%) create mode 100755 manager/scripts/container-run.py create mode 100755 manager/scripts/run-debug diff --git a/manager/integration/tests/basic_startup/knot-resolver.service b/manager/config/knot-resolver-manager.service similarity index 87% rename from manager/integration/tests/basic_startup/knot-resolver.service rename to manager/config/knot-resolver-manager.service index a0ebf5574..a1e6775a1 100644 --- a/manager/integration/tests/basic_startup/knot-resolver.service +++ b/manager/config/knot-resolver-manager.service @@ -9,4 +9,4 @@ ExecStart=/usr/bin/python3 -m knot_resolver_manager KillSignal=SIGINT [Install] -WantedBy=multi-user.target +WantedBy=multi-user.target \ No newline at end of file diff --git a/manager/containers/README.md b/manager/containers/README.md index ed0541591..4d799351f 100644 --- a/manager/containers/README.md +++ b/manager/containers/README.md @@ -4,4 +4,10 @@ All containers build configurations evolve from the `dev` container. If you want ## Naming -All containers defined here are named `knot-manager`. The directory name is the container's tag. \ No newline at end of file +All containers defined here are named `knot-manager`. The directory name is the container's tag. So, for example - `knot-manager:dev` is the reference development container. + +## Building + +```sh +for tag in +``` \ No newline at end of file diff --git a/manager/containers/debian/Containerfile b/manager/containers/debian/Containerfile index f7c76f872..01b1490ad 100644 --- a/manager/containers/debian/Containerfile +++ b/manager/containers/debian/Containerfile @@ -54,6 +54,9 @@ RUN apt-get update \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# Create knot-resolver-manager systemd service +COPY ./config/knot-resolver-manager.service /etc/systemd/system + # Copy only requirements, to cache them in docker layer # no poetry.lock, because here we have a different python version COPY ./pyproject.toml ./yarn.lock ./package.json /code/ diff --git a/manager/containers/dev/Containerfile b/manager/containers/dev/Containerfile index 8f34922fd..ac7985acb 100644 --- a/manager/containers/dev/Containerfile +++ b/manager/containers/dev/Containerfile @@ -63,6 +63,9 @@ RUN apt-get update \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* +# Create knot-resolver-manager systemd service +COPY ./config/knot-resolver-manager.service /etc/systemd/system + # Copy only requirements, to cache them in docker layer COPY ./poetry.lock ./pyproject.toml ./yarn.lock ./package.json /code/ diff --git a/manager/integration/runner.py b/manager/integration/runner.py index 0fc3af24a..7b739cb93 100644 --- a/manager/integration/runner.py +++ b/manager/integration/runner.py @@ -382,7 +382,7 @@ class TestRunner: # Temporary hack # build all test containers - ret = subprocess.call("poe build-containers", shell=True) + ret = subprocess.call("poe container-build", shell=True) assert ret == 0 # Run the tests diff --git a/manager/integration/tests/basic_startup/run b/manager/integration/tests/basic_startup/run index 959ccced9..126f038ed 100755 --- a/manager/integration/tests/basic_startup/run +++ b/manager/integration/tests/basic_startup/run @@ -5,9 +5,7 @@ set -e cd /test echo "Starting manager..." -cp knot-resolver.service /etc/systemd/system -systemctl daemon-reload -systemctl start knot-resolver.service +systemctl start knot-resolver-manager.service # give it time to start sleep 1 diff --git a/manager/integration/tests/worker_count/knot-resolver.service b/manager/integration/tests/worker_count/knot-resolver.service deleted file mode 100644 index a0ebf5574..000000000 --- a/manager/integration/tests/worker_count/knot-resolver.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Knot Resolver Manager -Requires=dbus -After=dbus - -[Service] -WorkingDirectory=/code -ExecStart=/usr/bin/python3 -m knot_resolver_manager -KillSignal=SIGINT - -[Install] -WantedBy=multi-user.target diff --git a/manager/integration/tests/worker_count/run b/manager/integration/tests/worker_count/run index c074d4d73..9d946f2e3 100755 --- a/manager/integration/tests/worker_count/run +++ b/manager/integration/tests/worker_count/run @@ -5,9 +5,7 @@ set -e cd /test echo "Starting manager..." -cp knot-resolver.service /etc/systemd/system -systemctl daemon-reload -systemctl start knot-resolver.service +systemctl start knot-resolver-manager.service # give it time to start sleep 1 diff --git a/manager/poetry.lock b/manager/poetry.lock index 92785059c..c83cf3e3a 100644 --- a/manager/poetry.lock +++ b/manager/poetry.lock @@ -159,6 +159,14 @@ category = "dev" optional = false python-versions = ">=3.6, <3.7" +[[package]] +name = "debugpy" +version = "1.2.1" +description = "An implementation of the Debug Adapter Protocol for Python" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + [[package]] name = "distlib" version = "0.3.1" @@ -946,7 +954,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.6.12" -content-hash = "939e224639141a5d45cc48343ffeb9f8daf50f4c856a2c43c0538ffa66423bc0" +content-hash = "e19083953f5e7f21555443d72701b765528ea26a5a9bbb746bb2093fb551acef" [metadata.files] aiohttp = [ @@ -1086,6 +1094,62 @@ dataclasses = [ {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, ] +debugpy = [ + {file = "debugpy-1.2.1-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:05726aa5fa7a65587ca0b1d58cbbf7f7b223300f989b152bd4dccc86a16e5c29"}, + {file = "debugpy-1.2.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:661d8d4b2632d952204237452d9764f20599fdd952cba49d6e985d3279453bc5"}, + {file = "debugpy-1.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:45864a250237128cfcb9b9b69ba844ef75fe7b7964691ce1a4c6af95d950cd5f"}, + {file = "debugpy-1.2.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ad42addff9b7f704ca4779bce5b40cf9cd9921172b60a9db10f94db28591b898"}, + {file = "debugpy-1.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:6c3592aa6441a0d4b976488d17d42a06531fe066f41e85b65a7dadc3480426bf"}, + {file = "debugpy-1.2.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:996f9070fa37fff1bc70438faea3aa82ad96b4794388bcd0151d21c9839cfbeb"}, + {file = "debugpy-1.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:b47a88b49d297a064d01045aa2f93cee230a4cbb42e0a4da2af9452175087906"}, + {file = "debugpy-1.2.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea36a3a71b3c7549710829bf1ef723222ebd6c9d43e7da0c30a87a8f718c7c38"}, + {file = "debugpy-1.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:cb6d1e65fc11f6e0bda65861a47ac49e8ad8e64bcc120f7d2f51cc8a0a36a2ba"}, + {file = "debugpy-1.2.1-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:45c81245cde0f47e3d6caaf82890f861a6147d0c6a0095c3ab1c578b50cbc1a8"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:8fa35cca6d724fde4ac7ecc680c36e4b6a51b7a575e967061a55057a8d55d98f"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7faa306b8dc21d79b329e2c757a6276f24621cbd4b37031b4ff5428160752e74"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:dabc631c63e4359995e9823d5e0f347d0b94fc8981a98ab41e2f670d2e7a2363"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:b93081a76f392a2f7363d0d0c0f3f1bf510d39745432339c165f4d5f8534e17d"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux2014_i686.whl", hash = "sha256:81d4eef9702a55af23fddf1a748a2b380ad7495a7d2a6a527dc18d23b39f089a"}, + {file = "debugpy-1.2.1-cp35-cp35m-manylinux2014_x86_64.whl", hash = "sha256:38f4bd04576632f1611cf3b8263e210f73652e39f31da63281199791fd6e9e5b"}, + {file = "debugpy-1.2.1-cp35-cp35m-win32.whl", hash = "sha256:ceb76da0c3551a2373503c3714d0ec77227e91f55715ecaff9890de40dd39bf7"}, + {file = "debugpy-1.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:eadedf204ec1921ea0e457292cecf0f8bf1df61aa1329073cacb8e6ce7bd1b82"}, + {file = "debugpy-1.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:9dbe15e36f15cbe8d8bdf488217e10c7548d0bd0e13dbfb29b90672bb3ee033a"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:97025c5485484a5e8fd30603d16e2ebbe170e976b10f72d31b7088872328ed83"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e5416bfcbc4d5cbcfff619b88fd2ee8624a75376bb6f348fc6f22e3a6dcf0384"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d8e063a734a5c2dc64e44e91c7d3f7282c9aebf630e0ea25d878f986bc20b8e7"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ff1a2887439ab0f2000b6a73d3fcdd8202bade433e190e5c0e8e1ad7b16969be"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:69109c0e1ec2ba6f91befb958c15b5a941a420feecb9e196a15ecc12a088745a"}, + {file = "debugpy-1.2.1-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:8dca68a83946ad18d4a60907d6cbfab2fd663f0818b1a575d4f18dc616787ac4"}, + {file = "debugpy-1.2.1-cp36-cp36m-win32.whl", hash = "sha256:779fa99a5c7a736b3f62db83200a325a81c1e2ae8961f5386c3318224531f474"}, + {file = "debugpy-1.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8861d8263c62be90bed696bf7b9d70be77a235c259035f5e2bcf0df2cf23b0d3"}, + {file = "debugpy-1.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:fa0ad34e3a13759bb0800ca93542bcc3294e19facfb4e8fc027720f388957b75"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:1da1f58e7ccbc7d2d5628347b522529b45390c8a58ff01912ea6cd0b4fbd4465"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3d6e779841246bb3512d630df5620828ae333618e8f93d117194a87904d1d2a6"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:b4ed4982b5d83e79f31417ab1feb5bd1bf7512cbd11ed396036197964dbcbf73"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:6e9f1aeb42820ae0bad07a0392cd5fb0b1da5aa0f4ada5d7e388e157892e5638"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:43846c3c986715d006ee15c6aaeba2c1b26d83a23da5f3d541d3f8eeae882062"}, + {file = "debugpy-1.2.1-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:8bf3d68592afe46439c5d35f3e1312e91882642b823e777b6415b8c150141437"}, + {file = "debugpy-1.2.1-cp37-cp37m-win32.whl", hash = "sha256:7fd7545475d74714531e36bcf77fbca6f123afa619be49c6b1543250d73de29a"}, + {file = "debugpy-1.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4d3ecffc93de06b67f14a4faf23f64c93a143b403ead989608c7a3f3b12a6859"}, + {file = "debugpy-1.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e42783d1731107433d421ee489aa71e76f521d1b6f25ead3d87fafdadc75d3fd"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e7cb1dd617dd429a0a0a2f6359bd6a5d08401715d530d318b449f6cbc78e1bdb"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:a196fc90f91f798c7b51e6acb74d01dd2ebd5b760b959058289e32b067407963"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:b1330eee0cbef2b056e7e9967800202e06bf1f8ef21fc215695aa315e76bab27"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ca9dd68bf3e52f60a628a191acb86e1b9654165a4327200b67008c3a05be6373"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:7520c4dc66e0230a7f4b5cbb336ea0abfecadaee0b7e0191803905f5bcef7c60"}, + {file = "debugpy-1.2.1-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:0ecff2d8bda1812434d71465189aa0fa6f2adf193955c39402aa25b510e563c0"}, + {file = "debugpy-1.2.1-cp38-cp38-win32.whl", hash = "sha256:23aca5c088a8389fb4bcddd16913608ebc06ce53cc2a09fc6eeb81f2d70249b4"}, + {file = "debugpy-1.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:1307ce1cd8cee2e7813c97764ac209abc8244af772ae299d1b81447b6b68b40c"}, + {file = "debugpy-1.2.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b76b3c43c4e4dbdd4940ac9c3d992c1ed100f33147d8824156c7e5bd11c71094"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:c388e332ce5c9592eafd925e9dec2dce889665cb45c4add15b0705053e9b3188"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3206b0ec848768fa083081469a705bc48b6941f90035f339ec7ef53bcdeb317c"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:185ff9ebcec6d644450c6df9d72a954b58710c2e6ed02b287e8dd610bd4bc7eb"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:53fed34ba47bd2ca9670e8942e64a000534742ee29d933eeae1563c6c126262e"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:c298fd4354c4f194c1df85b44bfccb0683504a86f3daa396533ae9b3a2eaa8b1"}, + {file = "debugpy-1.2.1-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:1740eeeb81b1f5779906a54306436b64c67919d4fbc1aee7b87367fc37f476ae"}, + {file = "debugpy-1.2.1-py2.py3-none-any.whl", hash = "sha256:6fd6e4d5295b6ac96f32053078688c6e4d8a0ed3d53cac5e14989303100c43aa"}, + {file = "debugpy-1.2.1.zip", hash = "sha256:3157a64db960dab322c4be6a44df3427ed965e4b618daba0c96cf71c13b82b61"}, +] distlib = [ {file = "distlib-0.3.1-py2.py3-none-any.whl", hash = "sha256:8c09de2c67b3e7deef7184574fc060ab8a793e7adbb183d942c389c8b13c52fb"}, {file = "distlib-0.3.1.zip", hash = "sha256:edf6116872c863e1aa9d5bb7cb5e05a022c519a4594dc703843343a9ddd9bff1"}, diff --git a/manager/pyproject.toml b/manager/pyproject.toml index dec95123e..ce7d98b61 100644 --- a/manager/pyproject.toml +++ b/manager/pyproject.toml @@ -28,15 +28,18 @@ requests = "^2.25.1" requests-unixsocket = "^0.2.0" click = "^7.1.2" toml = "^0.10.2" +debugpy = "^1.2.1" [tool.poe.tasks] run = { cmd = "python -m knot_resolver_manager", help = "Run the manager" } +run-debug = { cmd = "scripts/run-debug", help = "Run the manager under debugger" } test = { cmd = "pytest --cov=knot_resolver_manager --show-capture=all tests/", help = "Run tests" } check = { cmd = "scripts/codecheck", help = "Run static code analysis" } format = { cmd = "poetry run black knot_resolver_manager/ tests/", help = "Run 'Black' code formater" } fixdeps = { shell = "poetry install; yarn install", help = "Install/update dependencies according to configuration files"} commit = { shell = "scripts/commit", help = "Invoke every single check before commiting" } -build-containers = { shell = "scripts/build-containers", help = "Build all containers" } +container-build = { cmd = "scripts/container-buildall", help = "Build all containers" } +container-run = { cmd = "scripts/container-run.py", help = "Run a container" } clean = """ rm -rf .coverage .mypy_cache diff --git a/manager/scripts/build-containers b/manager/scripts/container-buildall similarity index 100% rename from manager/scripts/build-containers rename to manager/scripts/container-buildall diff --git a/manager/scripts/container-run.py b/manager/scripts/container-run.py new file mode 100755 index 000000000..049f85627 --- /dev/null +++ b/manager/scripts/container-run.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +import subprocess +from typing import List, Optional +import click +import time +import itertools + +PODMAN_EXECUTABLE = "/usr/bin/podman" + +def start_detached(image: str, publish: List[int] = []) -> str: + """Start a detached container""" + options = [ f"--publish={port}:{port}/tcp" for port in publish ] + command = ["podman", "run", "--rm", "-d", *options, image] + proc = subprocess.run(command, shell=False, executable=PODMAN_EXECUTABLE, stdout=subprocess.PIPE) + assert proc.returncode == 0 + return str(proc.stdout, 'utf8').strip() + +def exec(container_id: str, cmd: List[str]) -> int: + command = ["podman", "exec", container_id] + cmd + return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE) + +def exec_interactive(container_id: str, cmd: List[str]) -> int: + command = ["podman", "exec", "-ti", container_id] + cmd + return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE) + +def stop(container_id: str): + command = ["podman", "stop", container_id] + ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE) + assert ret == 0 + + +@click.command() +@click.argument("image", nargs=1) +@click.argument("command", nargs=-1) +@click.option("-p", "--publish", "publish", type=int, help="Port which should we publish") +def main(image: str, command: List[str], publish: Optional[int]): + # make sure arguments have the correct type + image = str(image) + command = list(command) + publish = [] if publish is None else [int(publish)] + + cont = start_detached(image, publish=publish) + # wait for the container to boot properly + time.sleep(0.5) + # run the command + ret = exec_interactive(cont, command) + # stop the container + stop(cont) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/manager/scripts/run-debug b/manager/scripts/run-debug new file mode 100755 index 000000000..c243d475e --- /dev/null +++ b/manager/scripts/run-debug @@ -0,0 +1,18 @@ +#!/bin/bash + +# fail early +set -e + +# ensure consistent behaviour +src_dir="$(dirname "$(realpath "$0")")" +source $src_dir/_env.sh + +# build all containers +poe container-build + +echo The debug server will be listening on port localhost:5678 +echo Use VSCode remote attach feature to connect to the debugger server +echo The manager will start after you connect +echo ---------------------------------------- + +poe container-run -p 5678 -- knot-manager:dev python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m knot_resolver_manager \ No newline at end of file -- 2.47.3