From 74884166c341eefe339ccac8ed41bedd2598b08b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ale=C5=A1=20Mr=C3=A1zek?= Date: Thu, 5 Feb 2026 14:42:10 +0100 Subject: [PATCH] python/knot_resolver: new basic structure --- pyproject.toml | 30 +++------- python/knot_resolver/__init__.py | 3 + python/knot_resolver/__main__.py | 4 ++ python/knot_resolver/client/__init__.py | 0 python/knot_resolver/client/__main__.py | 4 ++ python/knot_resolver/client/main.py | 2 + python/knot_resolver/controller/__init__.py | 5 ++ python/knot_resolver/controller/errors.py | 5 ++ .../knot_resolver/controller/supervisord.py | 40 +++++++++++++ python/knot_resolver/errors.py | 2 + python/knot_resolver/logging.py | 57 +++++++++++++++++++ python/knot_resolver/main.py | 44 ++++++++++++++ python/knot_resolver/manager/__init__.py | 0 python/knot_resolver/manager/__main__.py | 4 ++ python/knot_resolver/manager/errors.py | 5 ++ python/knot_resolver/manager/main.py | 2 + python/knot_resolver/manager/manager.py | 0 python/knot_resolver/resolver.py | 33 +++++++++++ python/knot_resolver/utils/__init__.py | 0 scripts/poe-tasks/check-code | 4 +- scripts/poe-tasks/fix-format | 2 +- scripts/poe-tasks/run | 8 +-- scripts/poe-tasks/test-unit | 2 +- setup.py | 30 +++------- tests/python/knot_resolver/test_logging.py | 8 +++ 25 files changed, 242 insertions(+), 52 deletions(-) create mode 100644 python/knot_resolver/__init__.py create mode 100644 python/knot_resolver/__main__.py create mode 100644 python/knot_resolver/client/__init__.py create mode 100644 python/knot_resolver/client/__main__.py create mode 100644 python/knot_resolver/client/main.py create mode 100644 python/knot_resolver/controller/__init__.py create mode 100644 python/knot_resolver/controller/errors.py create mode 100644 python/knot_resolver/controller/supervisord.py create mode 100644 python/knot_resolver/errors.py create mode 100644 python/knot_resolver/logging.py create mode 100644 python/knot_resolver/main.py create mode 100644 python/knot_resolver/manager/__init__.py create mode 100644 python/knot_resolver/manager/__main__.py create mode 100644 python/knot_resolver/manager/errors.py create mode 100644 python/knot_resolver/manager/main.py create mode 100644 python/knot_resolver/manager/manager.py create mode 100644 python/knot_resolver/resolver.py create mode 100644 python/knot_resolver/utils/__init__.py create mode 100644 tests/python/knot_resolver/test_logging.py diff --git a/pyproject.toml b/pyproject.toml index 585585cfe..c3b50ee13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,15 +1,14 @@ [tool.poetry] name = "knot-resolver" version = "6.2.0" -description = "Knot Resolver Manager - a Python program that automatically manages the other components of the resolver" +description = "A modern, high-performance, modular DNS resolver with DNSSEC validation and advanced policy." license = "GPL-3.0-or-later" -authors = [ - "Aleš Mrázek ", - "Václav Šraier " -] maintainers = [ "Aleš Mrázek " ] +authors = [ + "Václav Šraier " +] readme = "README.md" homepage = "https://www.knot-resolver.cz" repository = "https://gitlab.nic.cz/knot/knot-resolver" @@ -20,12 +19,6 @@ packages = [ ] exclude = ["**/*.in", "**/meson.build"] -# See currently open issue about building C extensions here: -# https://github.com/python-poetry/poetry/issues/2740 -[tool.poetry.build] -script = "build_c_extensions.py" -generate-setup-file = true - [tool.poetry.dependencies] python = "^3.8" aiohttp = "*" @@ -67,8 +60,9 @@ sphinx-rtd-theme = "^2.0.0" breathe = "^4.35.0" [tool.poetry.scripts] -kresctl = 'knot_resolver.client.main:main' -knot-resolver = 'knot_resolver.manager.main:main' +knot-resolver = "knot_resolver.main:main" +kres-manager = "knot_resolver.manager.main:main" +kresctl = "knot_resolver.client.main:main" [tool.poe.tasks] # Tasks to configure, build and run @@ -198,15 +192,7 @@ ignore = [ "TCH003", # *Move standard library import into a type-checking block ] exclude = [ - "tests/pytests/*", - "python/knot_resolver/controller/supervisord/plugin/*", - - # submodules - # "python/knot_resolver/client/*", - # "python/knot_resolver/client/*", - # "python/knot_resolver/controller/*", - # "python/knot_resolver/manager/*", - # "python/knot_resolver/utils/*", + "tests/python/knot_resolver*" ] [tool.ruff.lint.isort] diff --git a/python/knot_resolver/__init__.py b/python/knot_resolver/__init__.py new file mode 100644 index 000000000..fb1533f7c --- /dev/null +++ b/python/knot_resolver/__init__.py @@ -0,0 +1,3 @@ +from .constants import VERSION + +__version__ = VERSION diff --git a/python/knot_resolver/__main__.py b/python/knot_resolver/__main__.py new file mode 100644 index 000000000..40e2b013f --- /dev/null +++ b/python/knot_resolver/__main__.py @@ -0,0 +1,4 @@ +from .main import main + +if __name__ == "__main__": + main() diff --git a/python/knot_resolver/client/__init__.py b/python/knot_resolver/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/knot_resolver/client/__main__.py b/python/knot_resolver/client/__main__.py new file mode 100644 index 000000000..40e2b013f --- /dev/null +++ b/python/knot_resolver/client/__main__.py @@ -0,0 +1,4 @@ +from .main import main + +if __name__ == "__main__": + main() diff --git a/python/knot_resolver/client/main.py b/python/knot_resolver/client/main.py new file mode 100644 index 000000000..047ac14f6 --- /dev/null +++ b/python/knot_resolver/client/main.py @@ -0,0 +1,2 @@ +def main() -> None: + pass diff --git a/python/knot_resolver/controller/__init__.py b/python/knot_resolver/controller/__init__.py new file mode 100644 index 000000000..bb931fb6e --- /dev/null +++ b/python/knot_resolver/controller/__init__.py @@ -0,0 +1,5 @@ +from .supervisord import SupervisordController + +__all__ = [ + "SupervisordController", +] \ No newline at end of file diff --git a/python/knot_resolver/controller/errors.py b/python/knot_resolver/controller/errors.py new file mode 100644 index 000000000..2f9672d10 --- /dev/null +++ b/python/knot_resolver/controller/errors.py @@ -0,0 +1,5 @@ +from knot_resolver.errors import BaseKresError + + +class ControllerError(BaseKresError): + """Class for all errors used in controller submodules.""" diff --git a/python/knot_resolver/controller/supervisord.py b/python/knot_resolver/controller/supervisord.py new file mode 100644 index 000000000..80026c530 --- /dev/null +++ b/python/knot_resolver/controller/supervisord.py @@ -0,0 +1,40 @@ +import logging +import os +import shutil +import xmlrpc.client +from pathlib import Path + +import supervisor.xmlrpc + +from .errors import ControllerError + +logger = logging.getLogger(__name__) + + +class SupervisordController: + def __init__(self) -> None: + self._config = None + + def initialize(self) -> None: + logger.info("Initializing supervisord...") + logger.debug("Creating supervisord configuration") + + def exec(self) -> None: + supervisord = shutil.which("supervisord") + if not supervisord: + msg = "failed to find 'supervisord' executable" + raise ControllerError(msg) + + args = [ + "supervisord", + "--configuration", + str(Path("supervisord.conf").absolute()), + ] + + logger.debug("Execing supervisord...") + logging.shutdown() + os.execv(supervisord, args) # noqa: S606 + + async def shutdown(self) -> None: + pass + diff --git a/python/knot_resolver/errors.py b/python/knot_resolver/errors.py new file mode 100644 index 000000000..b88d9722a --- /dev/null +++ b/python/knot_resolver/errors.py @@ -0,0 +1,2 @@ +class BaseKresError(Exception): + """Base class for all errors used in Knot Resolver modules.""" diff --git a/python/knot_resolver/logging.py b/python/knot_resolver/logging.py new file mode 100644 index 000000000..8668125e7 --- /dev/null +++ b/python/knot_resolver/logging.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import logging +import logging.handlers +import sys +from typing import Any + +STDOUT = "stdout" +SYSLOG = "syslog" +STDERR = "stderr" + +NOTICE = (logging.WARNING + logging.INFO) // 2 + +_config_to_level = { + "critical": logging.CRITICAL, + "error": logging.ERROR, + "warning": logging.WARNING, + "notice": NOTICE, + "info": logging.INFO, + "debug": logging.DEBUG, +} + +_level_to_name = { + logging.CRITICAL: "CRITICAL", + logging.ERROR: "ERROR", + logging.WARNING: "WARNING", + NOTICE: "NOTICE", + logging.INFO: "INFO", + logging.DEBUG: "DEBUG", +} + +logging.addLevelName(NOTICE, _level_to_name[NOTICE]) + + +class KresLogger(logging.Logger): + def notice(self, message: str, *args: Any, **kwargs: Any) -> None: + if self.isEnabledFor(NOTICE): + self._log(NOTICE, message, args, **kwargs) + + +logging.setLoggerClass(KresLogger) + +logger = logging.getLogger(__name__) + + +def get_log_format() -> str: + return "[%(levelname)s] %(name)s: %(message)s" + + +def startup_logger(verbose: bool) -> None: + err_handler = logging.StreamHandler(sys.stderr) + err_handler.setFormatter(logging.Formatter(get_log_format())) + + logging_level = logging.DEBUG if verbose else NOTICE + logging.getLogger().setLevel(logging_level) + # Until we read the configuration, logging is to memory + logging.getLogger().addHandler(logging.handlers.MemoryHandler(10_000, logging.ERROR, err_handler)) diff --git a/python/knot_resolver/main.py b/python/knot_resolver/main.py new file mode 100644 index 000000000..57ef8bb62 --- /dev/null +++ b/python/knot_resolver/main.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import argparse + +from .constants import CONFIG_FILE, VERSION +from .logging import startup_logger +from .resolver import start_resolver + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description = "A modern, high-performance, modular DNS resolver with DNSSEC validation and advanced policy.", + ) + parser.add_argument( + "-V", + "--version", + help="Get version", + action="version", + version=VERSION, + ) + parser.add_argument( + "-v", + "--verbose", + help="Enable verbose (debug) logging", + action="store_true", + ) + parser.add_argument( + "-c", + "--config", + help="Optional, path to one or more configuration files (YAML/JSON).", + type=str, + nargs="+", + required=False, + default=[str(CONFIG_FILE)], + ) + return parser + + +def main() -> None: + parser = create_parser() + args = parser.parse_args() + + startup_logger(args.verbose) + start_resolver(config=args.config) diff --git a/python/knot_resolver/manager/__init__.py b/python/knot_resolver/manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/knot_resolver/manager/__main__.py b/python/knot_resolver/manager/__main__.py new file mode 100644 index 000000000..1063c0a51 --- /dev/null +++ b/python/knot_resolver/manager/__main__.py @@ -0,0 +1,4 @@ +from knot_resolver.manager.main import main + +if __name__ == "__main__": + main() diff --git a/python/knot_resolver/manager/errors.py b/python/knot_resolver/manager/errors.py new file mode 100644 index 000000000..5edf0ef51 --- /dev/null +++ b/python/knot_resolver/manager/errors.py @@ -0,0 +1,5 @@ +from knot_resolver.errors import BaseKresError + + +class KresManagerError(BaseKresError): + """Class for all errors used in manager submodules.""" diff --git a/python/knot_resolver/manager/main.py b/python/knot_resolver/manager/main.py new file mode 100644 index 000000000..047ac14f6 --- /dev/null +++ b/python/knot_resolver/manager/main.py @@ -0,0 +1,2 @@ +def main() -> None: + pass diff --git a/python/knot_resolver/manager/manager.py b/python/knot_resolver/manager/manager.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/knot_resolver/resolver.py b/python/knot_resolver/resolver.py new file mode 100644 index 000000000..b91847df5 --- /dev/null +++ b/python/knot_resolver/resolver.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import logging +from pathlib import Path + +from .controller import SupervisordController +from .controller.errors import ControllerError +from .utils.modeling.errors import DataModelingError + +logger = logging.getLogger(__name__) + + +def start_resolver(config: list[str]) -> int: + logger.notice("Starting Knot Resolver...") + + try: + # Ensure that configuration files paths do not change + # even when the working directory is changed. + config_paths = [Path(file).absolute() for file in config] + + logger.info("Loading configuration...") + + # load configuration here + + controller = SupervisordController() + controller.initialize() + controller.exec() + + except DataModelingError as e: + logger.exception("") + + except ControllerError as e: + logger.critical("") diff --git a/python/knot_resolver/utils/__init__.py b/python/knot_resolver/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/poe-tasks/check-code b/scripts/poe-tasks/check-code index 7ca26fb9f..0001fe566 100755 --- a/scripts/poe-tasks/check-code +++ b/scripts/poe-tasks/check-code @@ -8,7 +8,7 @@ source $src_dir/utils/_env.sh set +e # dirs to check -dirs="python/knot_resolver/ tests/manager scripts/poe-tasks/utils/create_setup.py" +dirs="python/knot_resolver/ tests/python/knot_resolver scripts/poe-tasks/utils/create_setup.py" # check imports echo -e "${yellow}Checking Python imports using Ruff...${reset}" @@ -24,7 +24,7 @@ echo # run static code analysis echo -e "${yellow}Running static code analysis using Ruff linter...${reset}" -ruff check python/knot_resolver/ tests/pytests +ruff check python/knot_resolver/ tests/python/knot_resolver check_rv $? echo diff --git a/scripts/poe-tasks/fix-format b/scripts/poe-tasks/fix-format index 8a8554a51..23c9f4f75 100755 --- a/scripts/poe-tasks/fix-format +++ b/scripts/poe-tasks/fix-format @@ -4,7 +4,7 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh -dirs="python/knot_resolver/ tests/manager scripts/poe-tasks/utils/create_setup.py" +dirs="python/knot_resolver/ tests/python/knot_resolver scripts/poe-tasks/utils/create_setup.py" # sort python import echo -e "${yellow}Sorting Python imports using ruff...${reset}" diff --git a/scripts/poe-tasks/run b/scripts/poe-tasks/run index 4d3ebc8cc..718b2bb91 100755 --- a/scripts/poe-tasks/run +++ b/scripts/poe-tasks/run @@ -29,7 +29,7 @@ shopt -u globstar shopt -u nullglob echo -echo -------------------------------------- -echo Starting Knot Resolver wit the Manager -echo -------------------------------------- -python3 -m knot_resolver.manager --config "$KRES_DEV_CONFIG_FILE" "$@" +echo ---------------------- +echo Starting Knot Resolver +echo ---------------------- +python3 -m knot_resolver --config "$KRES_DEV_CONFIG_FILE" "$@" diff --git a/scripts/poe-tasks/test-unit b/scripts/poe-tasks/test-unit index 85fd6089a..9560374e0 100755 --- a/scripts/poe-tasks/test-unit +++ b/scripts/poe-tasks/test-unit @@ -5,4 +5,4 @@ src_dir="$(dirname "$(realpath "$0")")" source $src_dir/utils/_env.sh # run pytest -env PYTHONPATH=. pytest --junitxml=unit.junit.xml --cov=python/knot_resolver --show-capture=all tests/manager +env PYTHONPATH=. pytest --junitxml=unit.junit.xml --cov=python/knot_resolver --show-capture=all tests/python/knot_resolver diff --git a/setup.py b/setup.py index 87aec5d39..795f9ff93 100644 --- a/setup.py +++ b/setup.py @@ -5,24 +5,10 @@ package_dir = \ {'': 'python'} packages = \ -['knot_resolver', - 'knot_resolver.client', - 'knot_resolver.client.commands', - 'knot_resolver.controller', - 'knot_resolver.controller.supervisord', - 'knot_resolver.controller.supervisord.plugin', - 'knot_resolver.datamodel', - 'knot_resolver.datamodel.templates', - 'knot_resolver.datamodel.types', - 'knot_resolver.manager', - 'knot_resolver.manager.files', - 'knot_resolver.manager.metrics', - 'knot_resolver.utils', - 'knot_resolver.utils.compat', - 'knot_resolver.utils.modeling'] +['knot_resolver', 'knot_resolver.controller', 'knot_resolver.manager'] package_data = \ -{'': ['*'], 'knot_resolver.datamodel.templates': ['macros/*']} +{'': ['*']} install_requires = \ ['aiohttp', 'jinja2', 'pyyaml', 'supervisor', 'typing-extensions'] @@ -31,16 +17,17 @@ extras_require = \ {'prometheus': ['prometheus-client'], 'watchdog': ['watchdog']} entry_points = \ -{'console_scripts': ['knot-resolver = knot_resolver.manager.main:main', +{'console_scripts': ['knot-resolver = knot_resolver.main:main', + 'kres-manager = knot_resolver.manager.main:main', 'kresctl = knot_resolver.client.main:main']} setup_kwargs = { 'name': 'knot-resolver', 'version': '6.2.0', - 'description': 'Knot Resolver Manager - a Python program that automatically manages the other components of the resolver', + 'description': 'A modern, high-performance, modular DNS resolver with DNSSEC validation and advanced policy.', 'long_description': "# Knot Resolver\n\n[![Build Status](https://gitlab.nic.cz/knot/knot-resolver/badges/nightly/pipeline.svg?x)](https://gitlab.nic.cz/knot/knot-resolver/commits/nightly)\n[![Coverage Status](https://gitlab.nic.cz/knot/knot-resolver/badges/nightly/coverage.svg?x)](https://www.knot-resolver.cz/documentation/latest)\n[![Packaging status](https://repology.org/badge/tiny-repos/knot-resolver.svg)](https://repology.org/project/knot-resolver/versions)\n\nKnot Resolver is a full caching DNS resolver implementation. The core architecture is tiny and efficient, written in C and [LuaJIT][luajit], providing a foundation and a state-machine-like API for extension modules. There are three built-in modules - *iterator*, *validator* and *cache* - which provide the main functionality of the resolver. A few other modules are automatically loaded by default to extend the resolver's functionality.\n\nSince Knot Resolver version 6, it also includes a so-called [manager][manager]. It is a new component written in [Python][python] that hides the complexity of older versions and makes it more user friendly. For example, new features include declarative configuration in YAML format and HTTP API for dynamic changes in the resolver and more.\n\nKnot Resolver uses a [different scaling strategy][scaling] than the rest of the DNS resolvers - no threading, shared-nothing architecture (except MVCC cache which can be shared), which allows you to pin workers to available CPU cores and grow by self-replication. You can start and stop additional workers based on the contention without downtime, which is automated by the [manager][manager] by default.\n\nThe LuaJIT modules, support for DNS privacy and DNSSEC, and persistent cache with low memory footprint make it a great personal DNS resolver or a research tool to tap into DNS data. Strong filtering rules, and auto-configuration with etcd make it a great large-scale resolver solution. It also has strong support for DNS over TCP, in particular TCP Fast-Open, query pipelining and deduplication, and response reordering.\n\nFor more on using the resolver, see the [User Documentation][doc]. See the [Developer Documentation][doc-dev] for detailed architecture and development.\n\n## Packages\n\nThe latest stable packages for various distributions are available in our\n[upstream repository](https://pkg.labs.nic.cz/doc/?project=knot-resolver).\nFollow the installation instructions to add this repository to your system.\n\nKnot Resolver is also available from the following distributions' repositories:\n\n* [Fedora and Fedora EPEL](https://src.fedoraproject.org/rpms/knot-resolver)\n* [Debian stable](https://packages.debian.org/stable/knot-resolver),\n [Debian testing](https://packages.debian.org/testing/knot-resolver),\n [Debian unstable](https://packages.debian.org/sid/knot-resolver)\n* [Ubuntu](https://packages.ubuntu.com/jammy/knot-resolver)\n* [Arch Linux](https://archlinux.org/packages/extra/x86_64/knot-resolver/)\n* [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=knot-resolver)\n\n### Packaging\n\nThe project uses [`apkg`](https://gitlab.nic.cz/packaging/apkg) for packaging.\nSee [`distro/README.md`](distro/README.md) for packaging specific instructions.\n\n## Building from sources\n\nKnot Resolver mainly depends on [KnotDNS][knot-dns] libraries, [LuaJIT][luajit], [libuv][libuv] and [Python][python].\n\nSee the [Building project][build] documentation page for more information.\n\n## Running\n\nBy default, Knot Resolver comes with [systemd][systemd] integration and you just need to start its service. It requires no configuration changes to run a server on localhost.\n\n```\n# systemctl start knot-resolver\n```\n\nSee the documentation at [knot-resolver.cz/documentation/latest][doc] for more information.\n\n## Running the Docker image\n\nRunning the Docker image is simple and doesn't require any dependencies or system modifications, just run:\n\n```\n$ docker run -Pit cznic/knot-resolver\n```\n\nThe images are meant as an easy way to try the resolver, and they're not designed for production use.\n\n## Contacting us\n\n- [GitLab issues](https://gitlab.nic.cz/knot/knot-resolver/issues) (you may authenticate via GitHub)\n- [mailing list](https://lists.nic.cz/postorius/lists/knot-resolver-announce.lists.nic.cz/)\n- [![Join the chat at https://gitter.im/CZ-NIC/knot-resolver](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/CZ-NIC/knot-resolver?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)\n\n[build]: https://www.knot-resolver.cz/documentation/latest/dev/build.html\n[doc]: https://www.knot-resolver.cz/documentation/latest/\n[doc-dev]: https://www.knot-resolver.cz/documentation/latest/dev\n[knot-dns]: https://www.knot-dns.cz/\n[luajit]: https://luajit.org/\n[libuv]: http://libuv.org\n[python]: https://www.python.org/\n[systemd]: https://systemd.io/\n[scaling]: https://www.knot-resolver.cz/documentation/latest/config-multiple-workers.html\n[manager]: https://www.knot-resolver.cz/documentation/latest/dev/architecture.html\n", - 'author': 'Aleš Mrázek', - 'author_email': 'ales.mrazek@nic.cz', + 'author': 'Václav Šraier', + 'author_email': 'vaclav.sraier@nic.cz', 'maintainer': 'Aleš Mrázek', 'maintainer_email': 'ales.mrazek@nic.cz', 'url': 'https://www.knot-resolver.cz', @@ -52,8 +39,7 @@ setup_kwargs = { 'entry_points': entry_points, 'python_requires': '>=3.8,<4.0', } -from build_c_extensions import * -build(setup_kwargs) + setup(**setup_kwargs) diff --git a/tests/python/knot_resolver/test_logging.py b/tests/python/knot_resolver/test_logging.py new file mode 100644 index 000000000..d99612f22 --- /dev/null +++ b/tests/python/knot_resolver/test_logging.py @@ -0,0 +1,8 @@ +import logging + +import knot_resolver.logging as kres_logging + + +def test_logger_notice() -> None: + logger = logging.getLogger(__name__) + logger.notice("this is NOTICE message") -- 2.47.3