From 10c657f6e3414a73e20b5c92924990640c55362b Mon Sep 17 00:00:00 2001 From: =?utf8?q?Ale=C5=A1?= Date: Thu, 6 May 2021 17:07:19 +0200 Subject: [PATCH] config: several config options and features - initial implementation of network.interfaces - network.edns-buffer-size accepts one or two arguments - suffixes for byte size and time interval --- manager/config/kres-manager.json | 42 +++++++----- manager/config/kres-manager.yaml | 36 +++++----- manager/config/kresd-template.j2 | 39 ++++++++--- .../tests/basic_crash/payload.json | 11 +-- .../tests/basic_startup/payload.json | 48 +++++++++---- .../tests/worker_count/payload.json | 11 +-- .../compat/dataclasses.py | 4 +- .../knot_resolver_manager/configuration.py | 41 ++++++++--- .../datamodel/cache_config.py | 14 ++++ .../knot_resolver_manager/datamodel/config.py | 18 ++--- .../datamodel/dns64_config.py | 4 +- .../datamodel/network_config.py | 68 ++++++++++++++++++- .../datamodel/options_config.py | 26 +++++++ .../knot_resolver_manager/datamodel/types.py | 40 ++++++++++- 14 files changed, 303 insertions(+), 99 deletions(-) diff --git a/manager/config/kres-manager.json b/manager/config/kres-manager.json index 8e18430ad..126ea1113 100644 --- a/manager/config/kres-manager.json +++ b/manager/config/kres-manager.json @@ -1,21 +1,31 @@ { - "server": { - "instances": 4 + "network": { + "interfaces": [ + { + "listen": "127.0.0.1" + }, + { + "listen": "127.0.0.1", + "kind": "dot" + }, + { + "listen": "::1", + "feebind": true + }, + { + "listen": "::1", + "kind": "dot", + "feebind": true + } + ], + "edns_buffer_size": { + "downstream": "4K" + } }, - "dns64": { - "prefix": "64:ff9b::" + "options": { + "prediction": true }, - "logging": { - "level": 4 - }, - "lua": { - "script": [ - "net.listen('127.0.0.1', 53, { kind = 'dns' })", - "net.listen('127.0.0.1', 853, { kind = 'tls' })", - "net.listen('::1', 53, { kind = 'dns', freebind = true })", - "net.listen('::1', 853, { kind = 'tls', freebind = true })", - "-- Cache size", - "cache.size = 100 * MB" - ] + "cache": { + "size_max": "100M" } } \ No newline at end of file diff --git a/manager/config/kres-manager.yaml b/manager/config/kres-manager.yaml index b5c16b2f1..3cde1367a 100644 --- a/manager/config/kres-manager.yaml +++ b/manager/config/kres-manager.yaml @@ -1,20 +1,16 @@ -server: - instances: 1 - -dns64: - prefix: "64:ff9b::" - -logging: - level: 4 - -lua: - script: | - """ - net.listen('127.0.0.1', 53, { kind = 'dns' }) - net.listen('127.0.0.1', 853, { kind = 'tls' }) - net.listen('::1', 53, { kind = 'dns', freebind = true }) - net.listen('::1', 853, { kind = 'tls', freebind = true }) - - -- Cache size - cache.size = 100 * MB - """ \ No newline at end of file +network: + interfaces: + - listen: 127.0.0.1 + - listen: 127.0.0.1 + kind: dot + - listen: ::1 + freebind: true + - listen: ::1 + kind: dot + freebind: true + edns_buffer_size: + downstream: 4K +options: + prediction: true +cache: + size_max: 100M diff --git a/manager/config/kresd-template.j2 b/manager/config/kresd-template.j2 index ac34262c2..b414df0cc 100644 --- a/manager/config/kresd-template.j2 +++ b/manager/config/kresd-template.j2 @@ -1,18 +1,41 @@ +{% if cfg.server.hostname %} +-- server.hostname +hostname('{{ cfg.server.hostname }}') +{% endif %} + +-- network.interfaces +{% for item in cfg.network.interfaces %} +net.listen('{{ item.get_address() }}', {{ item.get_port() if item.get_port() else 'nil' }}, { + kind = '{{ item.kind if item.kind != 'dot' else 'tls' }}', + freebind = {{ 'true' if item.freebind else 'false'}} +}) +{% endfor %} + +-- network.edns-buffer-size +net.bufsize({{ cfg.network.edns_buffer_size.get_downstream() }}, {{ cfg.network.edns_buffer_size.get_upstream() }}) + +-- modules modules = { 'hints > iterate', -- Load /etc/hosts and allow custom root hints", 'stats', -- Track internal statistics", - 'predict', -- Prefetch expiring/frequent records", -{%- if cfg.dns64 %} +{% if cfg.options.prediction %} + predict = { -- Prefetch expiring/frequent records" + window = {{ cfg.options.prediction.get_window() }}, + period = {{ cfg.options.prediction.period }} + }, +{% endif %} +{% if cfg.dns64 %} dns64 = '{{ cfg.dns64.prefix }}', -- dns64 -{%- endif %} +{% endif %} } -{%- if ( cfg.logging.level > 3 ) %} +-- cache +cache.open({{ cfg.cache.get_size_max() }}, 'lmdb://{{ cfg.cache.storage }}') + -- logging level -verbose(true) -{%- endif %} +verbose({{ 'true' if cfg.logging.level > 3 else 'false'}}) +{% if cfg.lua.script %} -- lua -{%- if cfg.lua.script %} {{ cfg.lua.script }} -{%- endif %} \ No newline at end of file +{% endif %} \ No newline at end of file diff --git a/manager/integration/tests/basic_crash/payload.json b/manager/integration/tests/basic_crash/payload.json index a90926c64..f0de8ce4e 100644 --- a/manager/integration/tests/basic_crash/payload.json +++ b/manager/integration/tests/basic_crash/payload.json @@ -2,14 +2,7 @@ "server": { "instances": %s }, - "lua": { - "script_list": [ - "net.listen('127.0.0.1', 53, { kind = 'dns' })", - "net.listen('127.0.0.1', 853, { kind = 'tls' })", - "net.listen('::1', 53, { kind = 'dns', freebind = true })", - "net.listen('::1', 853, { kind = 'tls', freebind = true })", - "-- Cache size", - "cache.size = 100 * MB" - ] + "logging": { + "level": 7 } } \ No newline at end of file diff --git a/manager/integration/tests/basic_startup/payload.json b/manager/integration/tests/basic_startup/payload.json index 74f185601..238fabde2 100644 --- a/manager/integration/tests/basic_startup/payload.json +++ b/manager/integration/tests/basic_startup/payload.json @@ -1,21 +1,41 @@ { "server": { - "instances": 1 + "hostname": "knot-resolver-manager", + "instances": 2, + "use_cache_gc": false }, - "dns64": { - "prefix": "64:ff9b::" + "network": { + "interfaces": [ + { + "listen": "127.0.0.1" + }, + { + "listen": "::1", + "freebind": true + }, + { + "listen": "127.0.0.1", + "kind": "dot" + }, + { + "listen": "::1", + "kind": "dot", + "freebind": true + } + ], + "edns_buffer_size": { + "downstream": "4K" + } }, - "logging": { - "level": 4 + "options": { + "prediction": true + }, + "dnssec": false, + "dns64": true, + "cache": { + "size_max": "10M" }, - "lua": { - "script_list": [ - "net.listen('127.0.0.1', 53, { kind = 'dns' })", - "net.listen('127.0.0.1', 853, { kind = 'tls' })", - "net.listen('::1', 53, { kind = 'dns', freebind = true })", - "net.listen('::1', 853, { kind = 'tls', freebind = true })", - "-- Cache size", - "cache.size = 100 * MB" - ] + "logging": { + "level": 7 } } \ No newline at end of file diff --git a/manager/integration/tests/worker_count/payload.json b/manager/integration/tests/worker_count/payload.json index a90926c64..f0de8ce4e 100644 --- a/manager/integration/tests/worker_count/payload.json +++ b/manager/integration/tests/worker_count/payload.json @@ -2,14 +2,7 @@ "server": { "instances": %s }, - "lua": { - "script_list": [ - "net.listen('127.0.0.1', 53, { kind = 'dns' })", - "net.listen('127.0.0.1', 853, { kind = 'tls' })", - "net.listen('::1', 53, { kind = 'dns', freebind = true })", - "net.listen('::1', 853, { kind = 'tls', freebind = true })", - "-- Cache size", - "cache.size = 100 * MB" - ] + "logging": { + "level": 7 } } \ No newline at end of file diff --git a/manager/knot_resolver_manager/compat/dataclasses.py b/manager/knot_resolver_manager/compat/dataclasses.py index 2bd9dee33..57f77fd2f 100644 --- a/manager/knot_resolver_manager/compat/dataclasses.py +++ b/manager/knot_resolver_manager/compat/dataclasses.py @@ -7,6 +7,6 @@ the option to do it transparently, without changing anything else. """ -from dataclasses import dataclass, is_dataclass +from dataclasses import dataclass, field, is_dataclass -__all__ = ["dataclass", "is_dataclass"] +__all__ = ["dataclass", "is_dataclass", "field"] diff --git a/manager/knot_resolver_manager/configuration.py b/manager/knot_resolver_manager/configuration.py index ad8b61b15..a13a38bb0 100644 --- a/manager/knot_resolver_manager/configuration.py +++ b/manager/knot_resolver_manager/configuration.py @@ -5,27 +5,50 @@ from jinja2 import Environment, Template from .datamodel import KresConfig _LUA_TEMPLATE_STR = """ +{% if cfg.server.hostname %} +-- server.hostname +hostname('{{ cfg.server.hostname }}') +{% endif %} + +-- network.interfaces +{% for item in cfg.network.interfaces %} +net.listen('{{ item.get_address() }}', {{ item.get_port() if item.get_port() else 'nil' }}, { + kind = '{{ item.kind if item.kind != 'dot' else 'tls' }}', + freebind = {{ 'true' if item.freebind else 'false'}} +}) +{% endfor %} + +-- network.edns-buffer-size +net.bufsize({{ cfg.network.edns_buffer_size.get_downstream() }}, {{ cfg.network.edns_buffer_size.get_upstream() }}) + +-- modules modules = { 'hints > iterate', -- Load /etc/hosts and allow custom root hints", 'stats', -- Track internal statistics", - 'predict', -- Prefetch expiring/frequent records", -{%- if cfg.dns64 %} +{% if cfg.options.prediction %} + predict = { -- Prefetch expiring/frequent records" + window = {{ cfg.options.prediction.get_window() }}, + period = {{ cfg.options.prediction.period }} + }, +{% endif %} +{% if cfg.dns64 %} dns64 = '{{ cfg.dns64.prefix }}', -- dns64 -{%- endif %} +{% endif %} } -{%- if ( cfg.logging.level > 3 ) %} +-- cache +cache.open({{ cfg.cache.get_size_max() }}, 'lmdb://{{ cfg.cache.storage }}') + -- logging level -verbose(true) -{%- endif %} +verbose({{ 'true' if cfg.logging.level > 3 else 'false'}}) +{% if cfg.lua.script %} -- lua -{%- if cfg.lua.script %} {{ cfg.lua.script }} -{%- endif %} +{% endif %} """ -_ENV = Environment(enable_async=True) +_ENV = Environment(enable_async=True, trim_blocks=True, lstrip_blocks=True) _LUA_TEMPLATE: Template = _ENV.from_string(_LUA_TEMPLATE_STR) diff --git a/manager/knot_resolver_manager/datamodel/cache_config.py b/manager/knot_resolver_manager/datamodel/cache_config.py index 8afa864c9..11620c8ed 100644 --- a/manager/knot_resolver_manager/datamodel/cache_config.py +++ b/manager/knot_resolver_manager/datamodel/cache_config.py @@ -1,8 +1,22 @@ +from typing import Optional + from knot_resolver_manager.compat.dataclasses import dataclass +from knot_resolver_manager.datamodel.types import SizeUnits from knot_resolver_manager.utils.dataclasses_parservalidator import DataclassParserValidatorMixin @dataclass class CacheConfig(DataclassParserValidatorMixin): + storage: str = "/var/cache/knot-resolver" + size_max: Optional[str] = None + _size_max_bytes: int = 100 * SizeUnits.mebibyte + + def __post_init__(self): + if self.size_max: + self._size_max_bytes = SizeUnits.parse(self.size_max) + + def get_size_max(self) -> int: + return self._size_max_bytes + def _validate(self): pass diff --git a/manager/knot_resolver_manager/datamodel/config.py b/manager/knot_resolver_manager/datamodel/config.py index 0b0543a06..9f16bb862 100644 --- a/manager/knot_resolver_manager/datamodel/config.py +++ b/manager/knot_resolver_manager/datamodel/config.py @@ -1,12 +1,10 @@ -from typing import Optional +from typing import Optional, Union from knot_resolver_manager.compat.dataclasses import dataclass from knot_resolver_manager.utils.dataclasses_parservalidator import DataclassParserValidatorMixin from .cache_config import CacheConfig from .dns64_config import Dns64Config -from .dnssec_config import DnssecConfig -from .hints_config import StaticHintsConfig from .logging_config import LoggingConfig from .lua_config import LuaConfig from .network_config import NetworkConfig @@ -18,14 +16,18 @@ from .server_config import ServerConfig class KresConfig(DataclassParserValidatorMixin): # pylint: disable=too-many-instance-attributes server: ServerConfig = ServerConfig() + network: NetworkConfig = NetworkConfig() options: OptionsConfig = OptionsConfig() - network: Optional[NetworkConfig] = None - static_hints: StaticHintsConfig = StaticHintsConfig() - dnssec: Optional[DnssecConfig] = None cache: CacheConfig = CacheConfig() - dns64: Optional[Dns64Config] = None + # DNS64 is disabled by default + dns64: Union[bool, Dns64Config] = False logging: LoggingConfig = LoggingConfig() - lua: LuaConfig = LuaConfig() + lua: Optional[LuaConfig] = None + + def __post_init__(self): + # if DNS64 is enabled with defaults + if self.dns64 is True: + self.dns64 = Dns64Config() def _validate(self): pass diff --git a/manager/knot_resolver_manager/datamodel/dns64_config.py b/manager/knot_resolver_manager/datamodel/dns64_config.py index fbad881c2..4d29c1c04 100644 --- a/manager/knot_resolver_manager/datamodel/dns64_config.py +++ b/manager/knot_resolver_manager/datamodel/dns64_config.py @@ -2,7 +2,7 @@ from knot_resolver_manager.compat.dataclasses import dataclass from knot_resolver_manager.utils.dataclasses_parservalidator import DataclassParserValidatorMixin from .errors import DataValidationError -from .types import IPV6_PREFIX_96 +from .types import RE_IPV6_PREFIX_96 @dataclass @@ -10,5 +10,5 @@ class Dns64Config(DataclassParserValidatorMixin): prefix: str = "64:ff9b::" def _validate(self): - if not bool(IPV6_PREFIX_96.match(self.prefix)): + if not bool(RE_IPV6_PREFIX_96.match(self.prefix)): raise DataValidationError("'dns64.prefix' must be valid IPv6 /96 prefix") diff --git a/manager/knot_resolver_manager/datamodel/network_config.py b/manager/knot_resolver_manager/datamodel/network_config.py index b60aedcd0..0acf66139 100644 --- a/manager/knot_resolver_manager/datamodel/network_config.py +++ b/manager/knot_resolver_manager/datamodel/network_config.py @@ -1,8 +1,74 @@ -from knot_resolver_manager.compat.dataclasses import dataclass +from typing import List, Optional, Union + +from knot_resolver_manager.compat.dataclasses import dataclass, field +from knot_resolver_manager.datamodel.types import SizeUnits from knot_resolver_manager.utils.dataclasses_parservalidator import DataclassParserValidatorMixin +@dataclass +class InterfacesConfig(DataclassParserValidatorMixin): + listen: str + kind: str = "dns" + freebind: bool = False + _address: Optional[str] = None + _port: Optional[int] = None + _kind_port_map = {"dns": 53, "xdp": 53, "dot": 853, "doh": 443} + + def __post_init__(self): + # split 'address@port' + if "@" in self.listen: + tmp = self.listen.split("@", maxsplit=1) + self._address = tmp[0] + self._port = int(tmp[1]) + # if port number not specified + self._address = self.listen + # set port number based on 'kind' + self._port = self._kind_port_map.get(self.kind) + + def get_address(self) -> Optional[str]: + return self._address + + def get_port(self) -> Optional[int]: + return self._port + + def _validate(self): + pass + + +@dataclass +class EdnsBufferSizeConfig(DataclassParserValidatorMixin): + downstream: Optional[str] = None + upstream: Optional[str] = None + _downstream_bytes: int = 1232 + _upstream_bytes: int = 1232 + + def __post_init__(self): + if self.downstream: + self._downstream_bytes = SizeUnits.parse(self.downstream) + if self.upstream: + self._upstream_bytes = SizeUnits.parse(self.upstream) + + def _validate(self): + pass + + def get_downstream(self) -> int: + return self._downstream_bytes + + def get_upstream(self) -> int: + return self._upstream_bytes + + @dataclass class NetworkConfig(DataclassParserValidatorMixin): + interfaces: List[InterfacesConfig] = field( + default_factory=lambda: [InterfacesConfig(listen="127.0.0.1"), InterfacesConfig(listen="::1", freebind=True)] + ) + edns_buffer_size: Union[str, EdnsBufferSizeConfig] = EdnsBufferSizeConfig() + + def __post_init__(self): + if isinstance(self.edns_buffer_size, str): + bufsize = self.edns_buffer_size + self.edns_buffer_size = EdnsBufferSizeConfig(downstream=bufsize, upstream=bufsize) + def _validate(self): pass diff --git a/manager/knot_resolver_manager/datamodel/options_config.py b/manager/knot_resolver_manager/datamodel/options_config.py index 31d0d5415..866b5b07b 100644 --- a/manager/knot_resolver_manager/datamodel/options_config.py +++ b/manager/knot_resolver_manager/datamodel/options_config.py @@ -1,8 +1,34 @@ +from typing import Optional, Union + from knot_resolver_manager.compat.dataclasses import dataclass +from knot_resolver_manager.datamodel.types import TimeUnits from knot_resolver_manager.utils.dataclasses_parservalidator import DataclassParserValidatorMixin +@dataclass +class PredictionConfig(DataclassParserValidatorMixin): + window: Optional[str] = None + _window_seconds: int = 15 * TimeUnits.minute + period: int = 24 + + def __post_init__(self): + if self.window: + self._window_seconds = TimeUnits.parse(self.window) + + def get_window(self) -> int: + return self._window_seconds + + def _validate(self): + pass + + @dataclass class OptionsConfig(DataclassParserValidatorMixin): + prediction: Union[bool, PredictionConfig] = False + + def __post_init__(self): + if self.prediction is True: + self.prediction = PredictionConfig() + def _validate(self): pass diff --git a/manager/knot_resolver_manager/datamodel/types.py b/manager/knot_resolver_manager/datamodel/types.py index e18335c1d..1d44250b7 100644 --- a/manager/knot_resolver_manager/datamodel/types.py +++ b/manager/knot_resolver_manager/datamodel/types.py @@ -1,3 +1,41 @@ import re -IPV6_PREFIX_96 = re.compile(r"^([0-9A-Fa-f]{1,4}:){2}:$") +from .errors import DataValidationError + +RE_IPV6_PREFIX_96 = re.compile(r"^([0-9A-Fa-f]{1,4}:){2}:$") + + +class TimeUnits: + second = 1 + minute = 60 + hour = 3600 + day = 24 * 3600 + + _re = re.compile(r"^(\d+)\s{0,1}([smhd]){0,1}$") + _map = {"s": second, "m": minute, "h": hour, "d": day} + + @staticmethod + def parse(time_str: str) -> int: + searched = TimeUnits._re.search(time_str) + if searched: + value, unit = searched.groups() + return int(value) * TimeUnits._map.get(unit, 1) + raise DataValidationError(f"failed to parse: {time_str}") + + +class SizeUnits: + byte = 1 + kibibyte = 1024 + mebibyte = 1024 ** 2 + gibibyte = 1024 ** 3 + + _re = re.compile(r"^([0-9]+)\s{0,1}([BKMG]){0,1}$") + _map = {"B": byte, "K": kibibyte, "M": mebibyte, "G": gibibyte} + + @staticmethod + def parse(size_str: str) -> int: + searched = SizeUnits._re.search(size_str) + if searched: + value, unit = searched.groups() + return int(value) * SizeUnits._map.get(unit, 1) + raise DataValidationError(f"failed to parse: {size_str}") -- 2.47.3