From: Vasek Sraier <git@vakabus.cz>
Date: Fri, 26 Mar 2021 12:17:39 +0000 (+0100)
Subject: container tooling: poe run now runs in a container, containers no longer include... 
X-Git-Tag: v6.0.0a1~195
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cc070cdd9f9db9f00de2ff2e55dad3c8b2e3c1cd;p=thirdparty%2Fknot-resolver.git

container tooling: poe run now runs in a container, containers no longer include the source (we now use bind mount)
---

diff --git a/manager/containers/debian/Containerfile b/manager/containers/debian/Containerfile
index 01b1490ad..5fd349b94 100644
--- a/manager/containers/debian/Containerfile
+++ b/manager/containers/debian/Containerfile
@@ -73,7 +73,7 @@ RUN poetry --version \
   # and install the dependencies
   && poetry install --no-dev --no-interaction --no-ansi
 
-# Copy the remaining code
-COPY . /code
+# Here, we would copy the remaining code if we wanted to permanently keep it in the container. We don't do that, we use read-only bind mounts
+# COPY . /code
 
 CMD ["/bin/systemd"]
\ No newline at end of file
diff --git a/manager/containers/dev/Containerfile b/manager/containers/dev/Containerfile
index ac7985acb..d18c70762 100644
--- a/manager/containers/dev/Containerfile
+++ b/manager/containers/dev/Containerfile
@@ -78,7 +78,7 @@ RUN echo "Running in $KNOT_ENV" \
     --no-interaction --no-ansi \
   && if test "$KNOT_ENV" = "dev"; then yarn install; fi
 
-# Copy the remaining code
-COPY . /code
+# Here, we would copy the remaining code if we wanted to permanently keep it in the container. We don't do that, we use read-only bind mounts
+# COPY . /code
 
 CMD ["/bin/systemd"]
\ No newline at end of file
diff --git a/manager/integration/runner.py b/manager/integration/runner.py
index 7b739cb93..6955e2bd8 100644
--- a/manager/integration/runner.py
+++ b/manager/integration/runner.py
@@ -394,7 +394,7 @@ class TestRunner:
                     print(f"Skipping test {Colors.YELLOW}{test.name}{Colors.RESET}")
                     continue
 
-                test.run(manager)
+                test.run(manager, inspect_failed)
 
 
 if __name__ == "__main__":
diff --git a/manager/integration/tests/basic_startup/test.toml b/manager/integration/tests/basic_startup/test.toml
index 7a316d7b2..d6bd3880a 100644
--- a/manager/integration/tests/basic_startup/test.toml
+++ b/manager/integration/tests/basic_startup/test.toml
@@ -2,4 +2,5 @@ image = "knot-manager:debian"
 cmd = ["/test/run"]
 
 [mount]
-"/test" = "integration/tests/basic_startup"
\ No newline at end of file
+"/test" = "integration/tests/basic_startup"
+"/code" = "."
\ No newline at end of file
diff --git a/manager/integration/tests/worker_count/test.toml b/manager/integration/tests/worker_count/test.toml
index bbc2fb5bd..dcb25097e 100644
--- a/manager/integration/tests/worker_count/test.toml
+++ b/manager/integration/tests/worker_count/test.toml
@@ -2,4 +2,5 @@ image = "knot-manager:debian"
 cmd = ["/test/run"]
 
 [mount]
-"/test" = "integration/tests/worker_count"
\ No newline at end of file
+"/test" = "integration/tests/worker_count"
+"/code" = "."
\ No newline at end of file
diff --git a/manager/knot_resolver_manager/__main__.py b/manager/knot_resolver_manager/__main__.py
index 50bd9285c..575a02fa3 100644
--- a/manager/knot_resolver_manager/__main__.py
+++ b/manager/knot_resolver_manager/__main__.py
@@ -1,8 +1,15 @@
+from typing import Optional
+from pathlib import Path
+import sys
+
 from aiohttp import web
-from knot_resolver_manager.kresd_manager import KresdManager
+import click
 
+from .kresd_manager import KresdManager
+from .utils import ignore_exceptions
 from . import configuration
 
+# when changing this, change the help message in main()
 _SOCKET_PATH = "/tmp/manager.sock"
 
 
@@ -17,7 +24,14 @@ async def apply_config(request: web.Request) -> web.Response:
     return web.Response(text="OK")
 
 
-def main():
+@click.command()
+@click.argument("listen", type=str, nargs=1, required=False, default=None)
+def main(listen: Optional[str]):
+    """Knot Resolver Manager
+
+    [listen] ... numeric port or a path for a Unix domain socket, default is \"/tmp/manager.sock\"
+    """
+
     app = web.Application()
 
     # initialize KresdManager
@@ -32,9 +46,21 @@ def main():
     # configure routing
     app.add_routes([web.get("/", hello), web.post("/config", apply_config)])
 
-    # run forever
-    web.run_app(app, path=_SOCKET_PATH)
+    # run forever, listen at the appropriate place
+    maybe_port = ignore_exceptions(None, ValueError, TypeError)(int)(listen)
+    if listen is None:
+        web.run_app(app, path=_SOCKET_PATH)
+    elif maybe_port is not None:
+        web.run_app(app, port=maybe_port)
+    elif Path(listen).parent.exists():
+        web.run_app(app, path=listen)
+    else:
+        print(
+            "Failed to parse LISTEN argument. Not an integer, not a valid path to a file in an existing directory.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
 
 
 if __name__ == "__main__":
-    main()
+    main()  # pylint: disable=no-value-for-parameter
diff --git a/manager/knot_resolver_manager/compat/__init__.py b/manager/knot_resolver_manager/compat/__init__.py
index 2f41e5328..efb04f463 100644
--- a/manager/knot_resolver_manager/compat/__init__.py
+++ b/manager/knot_resolver_manager/compat/__init__.py
@@ -1,4 +1,5 @@
 from . import asyncio
+from . import dataclasses
 
 
-__all__ = ["asyncio"]
+__all__ = ["asyncio", "dataclasses"]
diff --git a/manager/knot_resolver_manager/utils/__init__.py b/manager/knot_resolver_manager/utils/__init__.py
index cf8fa8974..de056390c 100644
--- a/manager/knot_resolver_manager/utils/__init__.py
+++ b/manager/knot_resolver_manager/utils/__init__.py
@@ -5,4 +5,22 @@ from .dataclasses_yaml import (
 )
 
 
-__all__ = ["dataclass_strictyaml_schema", "dataclass_strictyaml", "StrictyamlParser"]
+def ignore_exceptions(default, *exception):
+    def decorator(func):
+        def f(*nargs, **nkwargs):
+            try:
+                return func(*nargs, **nkwargs)
+            except exception:
+                return default
+
+        return f
+
+    return decorator
+
+
+__all__ = [
+    "dataclass_strictyaml_schema",
+    "dataclass_strictyaml",
+    "StrictyamlParser",
+    "ignore_exceptions",
+]
diff --git a/manager/poetry.lock b/manager/poetry.lock
index c83cf3e3a..271f72302 100644
--- a/manager/poetry.lock
+++ b/manager/poetry.lock
@@ -128,7 +128,7 @@ python-versions = "*"
 name = "click"
 version = "7.1.2"
 description = "Composable command line interface toolkit"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
@@ -954,7 +954,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6.12"
-content-hash = "e19083953f5e7f21555443d72701b765528ea26a5a9bbb746bb2093fb551acef"
+content-hash = "84b5fb8bb68a208f7a3b4027815766c8c6c2e82681789eacacf5838199b14a3d"
 
 [metadata.files]
 aiohttp = [
diff --git a/manager/pyproject.toml b/manager/pyproject.toml
index ce7d98b61..fdf6b36a6 100644
--- a/manager/pyproject.toml
+++ b/manager/pyproject.toml
@@ -14,6 +14,7 @@ strictyaml = "^1.3.2"
 pydbus = "^0.6.0"
 PyGObject = "^3.38.0"
 Jinja2 = "^2.11.3"
+click = "^7.1.2"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
@@ -31,14 +32,14 @@ toml = "^0.10.2"
 debugpy = "^1.2.1"
 
 [tool.poe.tasks]
-run = { cmd = "python -m knot_resolver_manager", help = "Run the manager" }
+run = { cmd = "scripts/run", help = "Run the manager" }
 run-debug = { cmd = "scripts/run-debug", help = "Run the manager under debugger" }
 test = { cmd = "pytest --cov=knot_resolver_manager --show-capture=all tests/", help = "Run tests" }
 check = { cmd = "scripts/codecheck", help = "Run static code analysis" }
 format = { cmd = "poetry run black knot_resolver_manager/ tests/", help = "Run 'Black' code formater" }
 fixdeps = { shell = "poetry install; yarn install", help = "Install/update dependencies according to configuration files"}
 commit = { shell = "scripts/commit", help = "Invoke every single check before commiting" }
-container-build = { cmd = "scripts/container-buildall", help = "Build all containers" }
+container-build = { cmd = "scripts/container-build", help = "Build containers (no arguments = all, otherwise arguments are tags that should be built)" }
 container-run = { cmd = "scripts/container-run.py", help = "Run a container" }
 clean = """
   rm -rf .coverage
diff --git a/manager/pyrightconfig.json b/manager/pyrightconfig.json
new file mode 100644
index 000000000..61815ecdb
--- /dev/null
+++ b/manager/pyrightconfig.json
@@ -0,0 +1,8 @@
+{
+  "include": [
+    "knot_resolver_manager"
+  ],
+  "exclude": [
+    "knot_resolver_manager/compat/asyncio.py"
+  ]
+}
\ No newline at end of file
diff --git a/manager/scripts/container-buildall b/manager/scripts/container-build
similarity index 50%
rename from manager/scripts/container-buildall
rename to manager/scripts/container-build
index 4f4132a92..1e0a3ac1f 100755
--- a/manager/scripts/container-buildall
+++ b/manager/scripts/container-build
@@ -8,7 +8,14 @@ src_dir="$(dirname "$(realpath "$0")")"
 source $src_dir/_env.sh
 
 
-# build the actual containers
-for tag in $(find containers -maxdepth 1 -type d -printf '%f\n' | grep -v containers); do
+if test "$#" -eq 0; then
+    containers="$(find containers -maxdepth 1 -type d -printf '%f\n' | grep -v containers)"
+else
+    containers="$@"
+fi
+
+
+# build all configured containers
+for tag in $containers; do
     podman build -t "knot-manager:$tag" -f "containers/$tag/Containerfile" .
 done
\ No newline at end of file
diff --git a/manager/scripts/container-run.py b/manager/scripts/container-run.py
index 049f85627..a66a4ca85 100755
--- a/manager/scripts/container-run.py
+++ b/manager/scripts/container-run.py
@@ -1,46 +1,102 @@
 #!/usr/bin/env python
 
 import subprocess
-from typing import List, Optional
+from typing import Dict, List, Optional
 import click
 import time
-import itertools
+from pathlib import Path, PurePath
+import sys
 
 PODMAN_EXECUTABLE = "/usr/bin/podman"
 
-def start_detached(image: str, publish: List[int] = []) -> str:
+
+def start_detached(
+    image: str, publish: List[int] = [], ro_mounts: Dict[PurePath, PurePath] = {}
+) -> str:
     """Start a detached container"""
-    options = [ f"--publish={port}:{port}/tcp" for port in publish ]
+    options = [f"--publish={port}:{port}/tcp" for port in publish] + [
+        f"--mount=type=bind,source={str(src)},destination={str(dst)},ro=true"
+        for src, dst in ro_mounts.items()
+    ]
     command = ["podman", "run", "--rm", "-d", *options, image]
-    proc = subprocess.run(command, shell=False, executable=PODMAN_EXECUTABLE, stdout=subprocess.PIPE)
+    proc = subprocess.run(
+        command, shell=False, executable=PODMAN_EXECUTABLE, stdout=subprocess.PIPE
+    )
     assert proc.returncode == 0
-    return str(proc.stdout, 'utf8').strip()
+    return str(proc.stdout, "utf8").strip()
+
 
 def exec(container_id: str, cmd: List[str]) -> int:
     command = ["podman", "exec", container_id] + cmd
     return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
 
+
 def exec_interactive(container_id: str, cmd: List[str]) -> int:
     command = ["podman", "exec", "-ti", container_id] + cmd
     return subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
 
+
 def stop(container_id: str):
     command = ["podman", "stop", container_id]
     ret = subprocess.call(command, shell=False, executable=PODMAN_EXECUTABLE)
     assert ret == 0
 
 
+def _get_git_root() -> PurePath:
+    result = subprocess.run(
+        "git rev-parse --show-toplevel", shell=True, stdout=subprocess.PIPE
+    )
+    return PurePath(str(result.stdout, encoding="utf8").strip())
+
+
 @click.command()
 @click.argument("image", nargs=1)
 @click.argument("command", nargs=-1)
-@click.option("-p", "--publish", "publish", type=int, help="Port which should we publish")
-def main(image: str, command: List[str], publish: Optional[int]):
+@click.option(
+    "-p", "--publish", "publish", multiple=True, type=int, help="Port which should be published"
+)
+@click.option(
+    "-m",
+    "--mount",
+    "mount",
+    multiple=True,
+    nargs=1,
+    type=str,
+    help="Read-only bind mounts into the container, value /path/on/host:/path/in/container",
+)
+@click.option(
+    "-c",
+    "--code",
+    "mount_code",
+    default=False,
+    is_flag=True,
+    type=bool,
+    help="Shortcut to mount gitroot into /code",
+)
+def main(
+    image: str,
+    command: List[str],
+    publish: Optional[int],
+    mount: Optional[List[str]],
+    mount_code: bool,
+):
     # make sure arguments have the correct type
     image = str(image)
     command = list(command)
-    publish = [] if publish is None else [int(publish)]
+    publish = [] if publish is None else [int(p) for p in publish]
+    mount = [] if mount is None else [x.split(":") for x in mount]
+    mount_path = {Path(x[0]).absolute(): Path(x[1]).absolute() for x in mount}
+    for src_path in mount_path:
+        if not src_path.exists():
+            print(
+                f'The specified path "{str(src_path)}" does not exist on the host system',
+                file=sys.stderr,
+            )
+            exit(1)
+    if mount_code:
+        mount_path[_get_git_root()] = Path("/code")
 
-    cont = start_detached(image, publish=publish)
+    cont = start_detached(image, publish=publish, ro_mounts=mount_path)
     # wait for the container to boot properly
     time.sleep(0.5)
     # run the command
@@ -48,5 +104,6 @@ def main(image: str, command: List[str], publish: Optional[int]):
     # stop the container
     stop(cont)
 
+
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()
diff --git a/manager/scripts/run b/manager/scripts/run
new file mode 100755
index 000000000..1ed740457
--- /dev/null
+++ b/manager/scripts/run
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# fail early
+set -e
+
+# ensure consistent behaviour
+src_dir="$(dirname "$(realpath "$0")")"
+source $src_dir/_env.sh
+
+# build dev container
+poe container-build dev
+
+echo Knot Manager API is accessible on http://localhost:9000
+echo -------------------------------------------------------
+
+poe container-run --code -p 9000 -- knot-manager:dev python -m knot_resolver_manager 9000
\ No newline at end of file
diff --git a/manager/scripts/run-debug b/manager/scripts/run-debug
index c243d475e..b175a4116 100755
--- a/manager/scripts/run-debug
+++ b/manager/scripts/run-debug
@@ -7,12 +7,12 @@ set -e
 src_dir="$(dirname "$(realpath "$0")")"
 source $src_dir/_env.sh
 
-# build all containers
-poe container-build
+# build dev container
+poe container-build dev
 
 echo The debug server will be listening on port localhost:5678
-echo Use VSCode remote attach feature to connect to the debugger server
+echo Use VSCode remote attach feature to connect to the debug server
 echo The manager will start after you connect
 echo ----------------------------------------
 
-poe container-run -p 5678 -- knot-manager:dev python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m knot_resolver_manager
\ No newline at end of file
+poe container-run -p 5678 --code -- knot-manager:dev python -m debugpy --listen 0.0.0.0:5678 --wait-for-client -m knot_resolver_manager
\ No newline at end of file