From: Aleš Mrázek Date: Tue, 10 Sep 2024 09:18:11 +0000 (+0200) Subject: python/knot_resolver: constants fixes X-Git-Tag: v6.0.9~9^2~18 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7b616ae31fc6365080b99e20d61a8c860b77d865;p=thirdparty%2Fknot-resolver.git python/knot_resolver: constants fixes --- diff --git a/python/knot_resolver/client/command.py b/python/knot_resolver/client/command.py index da42f2cf4..f87589bda 100644 --- a/python/knot_resolver/client/command.py +++ b/python/knot_resolver/client/command.py @@ -5,12 +5,7 @@ from pathlib import Path from typing import Dict, List, Optional, Tuple, Type, TypeVar from urllib.parse import quote -from knot_resolver.constants import ( - API_SOCK_PATH_DEFAULT, - API_SOCK_PATH_ENV_VAR, - CONFIG_FILE_PATH_DEFAULT, - CONFIG_FILE_PATH_ENV_VAR, -) +from knot_resolver.constants import API_SOCK_FILE, API_SOCK_FILE_ENV_VAR, CONFIG_FILE, CONFIG_FILE_ENV_VAR from knot_resolver.datamodel.types import IPAddressPort from knot_resolver.utils.modeling import parsing from knot_resolver.utils.modeling.exceptions import DataValidationError @@ -74,8 +69,8 @@ def determine_socket(namespace: argparse.Namespace) -> SocketDesc: if len(namespace.socket) > 0: return SocketDesc(namespace.socket[0], "--socket argument") - config_path = os.getenv(CONFIG_FILE_PATH_ENV_VAR) - socket_env = os.getenv(API_SOCK_PATH_ENV_VAR) + config_path = os.getenv(CONFIG_FILE_ENV_VAR) + socket_env = os.getenv(API_SOCK_FILE_ENV_VAR) socket: Optional[SocketDesc] = None # 2) socket from config file ('--config' argument) @@ -86,15 +81,15 @@ def determine_socket(namespace: argparse.Namespace) -> SocketDesc: socket = get_socket_from_config(Path(config_path), False) # 4) socket from environment variable elif socket_env: - socket = SocketDesc(socket_env, f'Environment variable "{API_SOCK_PATH_ENV_VAR}"') + socket = SocketDesc(socket_env, f'Environment variable "{API_SOCK_FILE_ENV_VAR}"') # 5) socket from config file (default config file constant) else: - socket = get_socket_from_config(CONFIG_FILE_PATH_DEFAULT, True) + socket = get_socket_from_config(CONFIG_FILE, True) if socket: return socket # 6) socket default - return SocketDesc(str(API_SOCK_PATH_DEFAULT), f'Default value "{API_SOCK_PATH_DEFAULT}"') + return SocketDesc(str(API_SOCK_FILE), f'Default value "{API_SOCK_FILE}"') class CommandArgs: diff --git a/python/knot_resolver/controller/supervisord/config_file.py b/python/knot_resolver/controller/supervisord/config_file.py index a9144505b..45a1a83ef 100644 --- a/python/knot_resolver/controller/supervisord/config_file.py +++ b/python/knot_resolver/controller/supervisord/config_file.py @@ -6,7 +6,7 @@ from typing import Literal from jinja2 import Template -from knot_resolver.constants import kres_cache_gc_executable, kresd_executable +from knot_resolver.constants import KRES_CACHE_GC_EXECUTABLE, KRESD_EXECUTABLE from knot_resolver.controller.interface import KresID, SubprocessType from knot_resolver.datamodel.config_schema import KresConfig from knot_resolver.datamodel.logging_schema import LogTargetEnum @@ -95,7 +95,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "gc.log", workdir=cwd, - command=f"{kres_cache_gc_executable()} -c {kres_cache_dir(config)}{kres_cache_gc_args(config)}", + command=f"{KRES_CACHE_GC_EXECUTABLE} -c {kres_cache_dir(config)}{kres_cache_gc_args(config)}", environment="", ) @@ -105,7 +105,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "policy-loader.log", workdir=cwd, - command=f"{kresd_executable()} -c {(policy_loader_config_file(config))} -c - -n", + command=f"{KRESD_EXECUTABLE} -c {(policy_loader_config_file(config))} -c - -n", environment="X-SUPERVISORD-TYPE=notify", ) @@ -115,7 +115,7 @@ class ProcessTypeConfig: return ProcessTypeConfig( # type: ignore[call-arg] logfile=supervisord_subprocess_log_dir(config) / "kresd%(process_num)d.log", workdir=cwd, - command=f"{kresd_executable()} -c {kresd_config_file_supervisord_pattern(config)} -n", + command=f"{KRESD_EXECUTABLE} -c {kresd_config_file_supervisord_pattern(config)} -n", environment='SYSTEMD_INSTANCE="%(process_num)d",X-SUPERVISORD-TYPE=notify', max_procs=int(config.max_workers) + 1, # +1 for the canary process ) diff --git a/python/knot_resolver/datamodel/config_schema.py b/python/knot_resolver/datamodel/config_schema.py index fe18516b6..4e3d8b3d1 100644 --- a/python/knot_resolver/datamodel/config_schema.py +++ b/python/knot_resolver/datamodel/config_schema.py @@ -3,7 +3,7 @@ import os import socket from typing import Any, Dict, List, Literal, Optional, Tuple, Union -from knot_resolver.constants import API_SOCK_PATH_DEFAULT, RUN_DIR_DEFAULT, VERSION, WORKERS_MAX_DEFAULT +from knot_resolver.constants import API_SOCK_FILE, RUN_DIR, VERSION from knot_resolver.datamodel.cache_schema import CacheSchema from knot_resolver.datamodel.dns64_schema import Dns64Schema from knot_resolver.datamodel.dnssec_schema import DnssecSchema @@ -24,6 +24,8 @@ from knot_resolver.utils.modeling import ConfigSchema from knot_resolver.utils.modeling.base_schema import lazy_default from knot_resolver.utils.modeling.exceptions import AggregateDataValidationError, DataValidationError +WORKERS_MAX = 256 + logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ def _workers_max_count() -> int: c = _cpu_count() if c: return c * 10 - return WORKERS_MAX_DEFAULT + return WORKERS_MAX def _get_views_tags(views: List[ViewSchema]) -> List[str]: @@ -109,10 +111,10 @@ class KresConfig(ConfigSchema): version: int = 1 nsid: Optional[EscapedStr] = None hostname: Optional[EscapedStr] = None - rundir: WritableDir = lazy_default(WritableDir, str(RUN_DIR_DEFAULT)) + rundir: WritableDir = lazy_default(WritableDir, str(RUN_DIR)) workers: Union[Literal["auto"], IntPositive] = IntPositive(1) - max_workers: IntPositive = IntPositive(WORKERS_MAX_DEFAULT) - management: ManagementSchema = lazy_default(ManagementSchema, {"unix-socket": str(API_SOCK_PATH_DEFAULT)}) + max_workers: IntPositive = IntPositive(WORKERS_MAX) + management: ManagementSchema = lazy_default(ManagementSchema, {"unix-socket": str(API_SOCK_FILE)}) webmgmt: Optional[WebmgmtSchema] = None options: OptionsSchema = OptionsSchema() network: NetworkSchema = NetworkSchema() @@ -178,7 +180,7 @@ class KresConfig(ConfigSchema): workers_max = _workers_max_count() if int(self.workers) > workers_max: raise ValueError( - f"can't run with more workers then the recommended maximum {workers_max} or hardcoded {WORKERS_MAX_DEFAULT}" + f"can't run with more workers then the recommended maximum {workers_max} or hardcoded {WORKERS_MAX}" ) # sanity check @@ -237,7 +239,7 @@ def get_rundir_without_validation(data: Dict[str, Any]) -> WritableDir: Used for initial manager startup. """ - return WritableDir(data["rundir"] if "rundir" in data else RUN_DIR_DEFAULT, object_path="/rundir") + return WritableDir(data["rundir"] if "rundir" in data else RUN_DIR, object_path="/rundir") def kres_config_json_schema() -> Dict[str, Any]: diff --git a/python/knot_resolver/datamodel/types/files.py b/python/knot_resolver/datamodel/types/files.py index 5f46c3a17..c29627296 100644 --- a/python/knot_resolver/datamodel/types/files.py +++ b/python/knot_resolver/datamodel/types/files.py @@ -6,7 +6,7 @@ from pathlib import Path from pwd import getpwnam from typing import Any, Dict, Tuple, Type, TypeVar -from knot_resolver.constants import GROUP_DEFAULT, USER_DEFAULT +from knot_resolver.constants import GROUP, USER from knot_resolver.datamodel.globals import get_resolve_root, get_strict_validation from knot_resolver.utils.modeling.base_value_type import BaseValueType @@ -157,8 +157,8 @@ def _kres_accessible(dest_path: Path, perm_mode: _PermissionMode) -> bool: _PermissionMode.EXECUTE: [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH], } - user_uid = getpwnam(USER_DEFAULT).pw_uid - user_gid = getgrnam(GROUP_DEFAULT).gr_gid + user_uid = getpwnam(USER).pw_uid + user_gid = getgrnam(GROUP).gr_gid dest_stat = os.stat(dest_path) dest_uid = dest_stat.st_uid @@ -195,7 +195,7 @@ class ReadableFile(File): super().__init__(source_value, parents=parents, object_path=object_path) if self.strict_validation and not _kres_accessible(self._value, _PermissionMode.READ): - raise ValueError(f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to read '{self._value}'") + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to read '{self._value}'") class WritableDir(Dir): @@ -213,9 +213,7 @@ class WritableDir(Dir): if self.strict_validation and not _kres_accessible( self._value, _PermissionMode.WRITE | _PermissionMode.EXECUTE ): - raise ValueError( - f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to write/execute '{self._value}'" - ) + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to write/execute '{self._value}'") class WritableFilePath(FilePath): @@ -234,6 +232,4 @@ class WritableFilePath(FilePath): if self.strict_validation and not _kres_accessible( self._value.parent, _PermissionMode.WRITE | _PermissionMode.EXECUTE ): - raise ValueError( - f"{USER_DEFAULT}:{GROUP_DEFAULT} has insufficient permissions to write/execute'{self._value.parent}'" - ) + raise ValueError(f"{USER}:{GROUP} has insufficient permissions to write/execute'{self._value.parent}'") diff --git a/python/knot_resolver/manager/constants.py b/python/knot_resolver/manager/constants.py index 580ac4d8e..43b1c04bc 100644 --- a/python/knot_resolver/manager/constants.py +++ b/python/knot_resolver/manager/constants.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from typing import TYPE_CHECKING, Optional @@ -7,6 +8,14 @@ if TYPE_CHECKING: from knot_resolver.manager.config_store import ConfigStore +LOGGING_LEVEL_STARTUP = logging.DEBUG +PID_FILE_NAME = "knot-resolver.pid" + +FIX_COUNTER_ATTEMPTS_MAX = 2 +FIX_COUNTER_DECREASE_INTERVAL_SEC = 30 * 60 +WATCHDOG_INTERVAL_SEC: float = 5 + + def kres_cache_dir(config: "KresConfig") -> Path: return config.cache.storage.to_path() diff --git a/python/knot_resolver/manager/logging.py b/python/knot_resolver/manager/logging.py index ff140dcd9..c9b446533 100644 --- a/python/knot_resolver/manager/logging.py +++ b/python/knot_resolver/manager/logging.py @@ -4,11 +4,12 @@ import os import sys from typing import Optional -from knot_resolver.constants import LOGGING_LEVEL_STARTUP from knot_resolver.datamodel.config_schema import KresConfig from knot_resolver.datamodel.logging_schema import LogTargetEnum from knot_resolver.manager.config_store import ConfigStore, only_on_real_changes_update +from .constants import LOGGING_LEVEL_STARTUP + logger = logging.getLogger(__name__) diff --git a/python/knot_resolver/manager/main.py b/python/knot_resolver/manager/main.py index 0339790b4..294e6cb9c 100644 --- a/python/knot_resolver/manager/main.py +++ b/python/knot_resolver/manager/main.py @@ -9,7 +9,7 @@ import sys from pathlib import Path from typing import NoReturn -from knot_resolver.constants import CONFIG_FILE_PATH_DEFAULT, CONFIG_FILE_PATH_ENV_VAR, VERSION +from knot_resolver.constants import CONFIG_FILE, CONFIG_FILE_ENV_VAR, VERSION from knot_resolver.manager.logging import logger_startup from knot_resolver.manager.server import start_server from knot_resolver.utils import compat @@ -27,7 +27,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "-c", "--config", - help="Config file to load. Overrides default config location at '" + str(CONFIG_FILE_PATH_ENV_VAR) + "'", + help="Config file to load. Overrides default config location at '" + str(CONFIG_FILE) + "'", type=str, nargs=1, required=False, @@ -44,13 +44,13 @@ def main() -> NoReturn: args = parse_args() # where to look for config - config_env = os.getenv(CONFIG_FILE_PATH_ENV_VAR) + config_env = os.getenv(CONFIG_FILE_ENV_VAR) if args.config is not None: config_path = Path(args.config[0]) elif config_env is not None: config_path = Path(config_env) else: - config_path = CONFIG_FILE_PATH_DEFAULT + config_path = CONFIG_FILE exit_code = compat.asyncio.run(start_server(config=config_path)) sys.exit(exit_code) diff --git a/python/knot_resolver/manager/manager.py b/python/knot_resolver/manager/manager.py index 114ad68e8..74d90ec32 100644 --- a/python/knot_resolver/manager/manager.py +++ b/python/knot_resolver/manager/manager.py @@ -6,7 +6,6 @@ from secrets import token_hex from subprocess import SubprocessError from typing import Any, Callable, List, Optional -from knot_resolver.constants import FIX_COUNTER_ATTEMPTS_MAX, FIX_COUNTER_DECREASE_INTERVAL_SEC, WATCHDOG_INTERVAL_SEC from knot_resolver.controller.exceptions import SubprocessControllerException from knot_resolver.controller.interface import Subprocess, SubprocessController, SubprocessStatus, SubprocessType from knot_resolver.controller.registered_workers import command_registered_workers, get_registered_workers_kresids @@ -16,6 +15,8 @@ from knot_resolver.utils.compat.asyncio import create_task from knot_resolver.utils.functional import Result from knot_resolver.utils.modeling.types import NoneType +from .constants import FIX_COUNTER_ATTEMPTS_MAX, FIX_COUNTER_DECREASE_INTERVAL_SEC, WATCHDOG_INTERVAL_SEC + logger = logging.getLogger(__name__) diff --git a/python/knot_resolver/manager/server.py b/python/knot_resolver/manager/server.py index ea2b419c0..972b167fb 100644 --- a/python/knot_resolver/manager/server.py +++ b/python/knot_resolver/manager/server.py @@ -17,7 +17,7 @@ from aiohttp.web_app import Application from aiohttp.web_response import json_response from aiohttp.web_runner import AppRunner, TCPSite, UnixSite -from knot_resolver.constants import CONFIG_FILE_PATH_DEFAULT, PID_FILE_NAME +from knot_resolver.constants import CONFIG_FILE from knot_resolver.controller import get_best_controller_implementation from knot_resolver.controller.exceptions import SubprocessControllerExecException from knot_resolver.controller.registered_workers import command_single_registered_worker @@ -40,7 +40,7 @@ from knot_resolver.utils.modeling.types import NoneType from knot_resolver.utils.systemd_notify import systemd_notify from .config_store import ConfigStore -from .constants import init_user_constants +from .constants import PID_FILE_NAME, init_user_constants from .exceptions import KresManagerException from .logging import logger_init from .manager import KresManager @@ -506,7 +506,7 @@ async def _sigterm_while_shutting_down(): sys.exit(128 + signal.SIGTERM) -async def start_server(config: Path = CONFIG_FILE_PATH_DEFAULT) -> int: +async def start_server(config: Path = CONFIG_FILE) -> int: # This function is quite long, but it describes how manager runs. So let's silence pylint # pylint: disable=too-many-statements