from http import HTTPStatus
from pathlib import Path
from time import time
-from typing import Any, Optional, Set, Union
+from typing import Any, Optional, Set, Union, cast
-from aiohttp import web
+from aiohttp import ETag, web
from aiohttp.web import middleware
from aiohttp.web_app import Application
from aiohttp.web_response import json_response
from aiohttp.web_runner import AppRunner, TCPSite, UnixSite
+from typing_extensions import Literal
import knot_resolver_manager.utils.custom_atexit as atexit
from knot_resolver_manager import log, statistics
)
@statistics.async_timing_histogram(statistics.MANAGER_REQUEST_RECONFIGURE_LATENCY)
- async def _handler_apply_config(self, request: web.Request) -> web.Response:
+ async def _handler_config_query(self, request: web.Request) -> web.Response:
"""
Route handler for changing resolver configuration
"""
# parse the incoming data
document_path = request.match_info["path"]
+ etags = request.if_match
last: ParsedTree = self.config_store.get().get_unparsed_data()
- new_partial: ParsedTree = parse(await request.text(), request.content_type)
- config = last.update(document_path, new_partial)
+ update_with: ParsedTree = parse(await request.text(), request.content_type)
+
+ if etags is not None and last.etag not in map(str, etags):
+ return web.Response(status=HTTPStatus.PRECONDITION_FAILED)
+
+ op = cast(Literal["get", "post", "delete", "patch", "put"], request.method.lower())
+ root, to_return = last.query(op, document_path, update_with)
# validate config
- config_validated = KresConfig(config)
+ config_validated = KresConfig(root)
# apply config
await self.config_store.update(config_validated)
# return success
- return web.Response()
+ res = web.Response(status=HTTPStatus.OK, text=str(to_return))
+ res.etag = ETag(config_validated.get_unparsed_data().etag)
+ return res
async def _handler_metrics(self, _request: web.Request) -> web.Response:
return web.Response(
self.app.add_routes(
[
web.get("/", self._handler_index),
- web.post(r"/config{path:.*}", self._handler_apply_config),
+ web.post(r"/config{path:.*}", self._handler_config_query),
+ web.put(r"/config{path:.*}", self._handler_config_query),
+ web.patch(r"/config{path:.*}", self._handler_config_query),
+ web.get(r"/config{path:.*}", self._handler_config_query),
+ web.delete(r"/config{path:.*}", self._handler_config_query),
web.post("/stop", self._handler_stop),
web.get("/schema", self._handler_schema),
web.get("/schema/ui", self._handle_view_schema),
from .base_schema import BaseSchema
from .base_value_type import BaseValueType
from .parsing import ParsedTree, parse, parse_json, parse_yaml
+from .query import QueryTree
__all__ = [
"BaseValueType",
"parse",
"parse_yaml",
"parse_json",
+ "QueryTree",
]
-import copy
+import base64
import json
-import re
from enum import Enum, auto
+from hashlib import blake2b
from typing import Any, Dict, List, Optional, Set, Tuple, Union
import yaml
+from typing_extensions import Literal
from yaml.constructor import ConstructorError
from yaml.nodes import MappingNode
+from knot_resolver_manager.utils.modeling.query import QueryTree
+
from .exceptions import DataParsingError
-from .types import is_internal_field_name
class ParsedTree:
return name.replace("-", "_")
return name
- def __init__(self, data: Union[Dict[str, Any], str, int, bool]):
+ def __init__(self, data: Union[Dict[str, Any], str, int, bool, List[Any]]):
self._data = data
- def to_raw(self) -> Union[Dict[str, Any], str, int, bool]:
+ def to_raw(self) -> Union[Dict[str, Any], str, int, bool, List[Any]]:
return self._data
def __getitem__(self, key: str) -> Any:
assert isinstance(self._data, dict)
return {ParsedTree._convert_external_field_name_to_internal(key) for key in self._data.keys()}
- _SUBTREE_MUTATION_PATH_PATTERN = re.compile(r"^(/[^/]+)*/?$")
-
- def update(self, path: str, data: "ParsedTree") -> "ParsedTree":
-
- # prepare and validate the path object
- path = path[:-1] if path.endswith("/") else path
- if re.match(ParsedTree._SUBTREE_MUTATION_PATH_PATTERN, path) is None:
- raise DataParsingError("Provided object path for mutation is invalid.")
- if "_" in path:
- raise DataParsingError("Provided object path contains character '_', which is illegal")
- # Note: mutation happens on the internal dict only, therefore we are working with external
- # naming only. That means, there are '-' in between words.
- path = path[1:] if path.startswith("/") else path
-
- # now, the path variable should contain '/' separated field names
-
- # check if we should mutate whole object
- if path == "":
- return data
-
- # find the subtree we will replace in a copy of the original object
- to_mutate = copy.deepcopy(self.to_raw())
- obj = to_mutate
- parent = None
-
- for segment in path.split("/"):
- assert isinstance(obj, dict)
-
- if segment == "":
- raise DataParsingError(f"Unexpectedly empty segment in path '{path}'")
- elif is_internal_field_name(segment):
- raise DataParsingError(
- "No, changing internal fields (starting with _) is not allowed. Nice try though."
- )
- elif segment in obj:
- parent = obj
- obj = obj[segment]
- elif segment not in obj:
- parent = obj
- obj = {}
- parent[segment] = obj
- assert parent is not None
-
- # assign the subtree
- last_name = path.split("/")[-1]
- parent[last_name] = data.to_raw()
-
- return ParsedTree(to_mutate)
+ def query(
+ self,
+ op: Literal["get", "post", "delete", "patch", "put"],
+ path: str,
+ update_with: Optional["ParsedTree"] = None,
+ ) -> "Tuple[ParsedTree, Optional[ParsedTree]]":
+ new_root, res = QueryTree(self._data).query(
+ op, path, update_with=QueryTree(update_with.to_raw()) if update_with is not None else None
+ )
+ return ParsedTree(new_root.to_raw()), ParsedTree(res.to_raw()) if res is not None else None
+
+ @property
+ def etag(self) -> str:
+ m = blake2b(digest_size=9)
+ m.update(json.dumps(self._data, sort_keys=True).encode("utf8"))
+ return base64.urlsafe_b64encode(m.digest()).decode("utf8")
# custom hook for 'json.loads()' to detect duplicate keys in data
--- /dev/null
+import copy
+import json
+import re
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
+
+from typing_extensions import Literal
+
+from knot_resolver_manager.utils.modeling.exceptions import DataParsingError
+
+
+class QueryTree:
+ """
+ Simple wrapper for raw data which allows modification queries to be run on top.
+
+ IMMUTABLE, DO NOT MODIFY
+ """
+
+ def is_scalar(self) -> bool:
+ """
+ true if the object represents a primitive type
+ """
+ return isinstance(self._data, (str, int, bool))
+
+ def is_object(self) -> bool:
+ """
+ true if the object represents a list or dict
+ """
+ return isinstance(self._data, (list, dict))
+
+ def _is_list(self) -> bool:
+ return isinstance(self._data, list)
+
+ def _is_dict(self) -> bool:
+ return isinstance(self._data, dict)
+
+ def _upsert(self, key: str, value: "QueryTree") -> None:
+ """
+ WARNING!!! MUTATES THE TREE
+
+ update or insert
+ """
+ if isinstance(self._data, dict):
+ self._data[key] = value.to_raw()
+ elif isinstance(self._data, list):
+ if key in self:
+ self._data[int(key)] = value.to_raw()
+ else:
+ raise DataParsingError("query invalid: can't set a value of an item in a list at a non-existent index")
+ else:
+ assert False, "this should never happen"
+
+ def _append(self, value: "QueryTree") -> None:
+ """
+ WARNING!!! MUTATES THE TREE
+
+ append to a list
+ """
+ assert isinstance(self._data, list)
+ self._data.append(value.to_raw())
+
+ def _delete(self, key: str) -> None:
+ """
+ WARNING!!! MUTATES THE TREE
+
+ deletes a key
+ """
+ assert self.is_object()
+ if isinstance(self._data, list):
+ del self._data[int(key)]
+ elif isinstance(self._data, dict):
+ del self._data[key]
+ else:
+ assert False, "never happens"
+
+ def value(self) -> Union[str, int, bool]:
+ if self.is_object():
+ raise DataParsingError("attempted to access object as a scalar")
+
+ assert isinstance(self._data, (str, int, bool)) # make type checker happy
+ return self._data
+
+ def __init__(self, data: Union[Dict[str, Any], str, int, bool, List[Any]]):
+ self._data = data
+
+ def to_raw(self) -> Union[Dict[str, Any], str, int, bool, List[Any]]:
+ return self._data
+
+ def __getitem__(self, key: Union[str, int]) -> "QueryTree":
+ if self.is_scalar():
+ raise DataParsingError(f"attempted to access scalar value '{self._data}' as an object type")
+
+ if isinstance(self._data, list):
+ return QueryTree(self._data[int(key)])
+ elif isinstance(self._data, dict):
+ return QueryTree(self._data[str(key)])
+ else:
+ raise RuntimeError("unexpected type in self._data, this should never happen")
+
+ def __contains__(self, key: Union[str, int]) -> bool:
+ if self.is_scalar():
+ raise DataParsingError(f"attempted to access scalar value '{self._data}' as an object type")
+
+ if isinstance(self._data, list):
+ return int(key) < len(self._data)
+ elif isinstance(self._data, dict):
+ return key in self._data
+ else:
+ raise RuntimeError("unexpected type in self._data, this should never happen")
+
+ def __str__(self) -> str:
+ return json.dumps(self._data, sort_keys=False, indent=2)
+
+ def keys(self) -> Set[Any]:
+ if self.is_scalar():
+ raise DataParsingError(f"attempted to access scalar value '{self._data}' as an object type")
+
+ if isinstance(self._data, dict):
+ return set(self._data.keys())
+ elif isinstance(self._data, list):
+ return set(range(len(self._data)))
+ else:
+ raise RuntimeError("unexpected type in self._data, this should never happen")
+
+ _SUBTREE_MUTATION_PATH_PATTERN = re.compile(r"^(/[^/]+)*/?$")
+
+ def _preprocess_query_path(self, path: str) -> str:
+ # prepare and validate the path object
+ path = path[:-1] if path.endswith("/") else path
+ if re.match(QueryTree._SUBTREE_MUTATION_PATH_PATTERN, path) is None:
+ raise DataParsingError("Provided object path for mutation is invalid.")
+ if "_" in path:
+ raise DataParsingError("Provided object path contains character '_', which is illegal")
+
+ # now, the path variable should contain '/' separated field names
+ return path.strip("/")
+
+ def _copy_and_find(self, path: str) -> Tuple["QueryTree", "QueryTree", Optional["QueryTree"], str]:
+ """
+ Returns (fakeroot, parent, Optional[queryTarget])
+
+ - fakeroot has the real root in a field called 'root'
+ - queryTarget is None, when it refers to non-existent object
+ """
+
+ path = self._preprocess_query_path(path)
+
+ # `self` is considered immutable, do all operations on a copy
+ rwcopy = copy.deepcopy(self)
+ # make a fake root, so that we do not have to handle special cases for root node
+ rwcopy._data = {"root": rwcopy._data} # pylint: disable=protected-access
+ segments = f"root/{path}".strip("/").split("/")
+
+ # walk the tree
+ obj: Optional[QueryTree] = rwcopy
+ parent: QueryTree = rwcopy
+ segment = "" # just to make type checker happy
+ for segment in segments:
+ assert len(segment) > 0
+ if obj is None:
+ raise DataParsingError(
+ f"query path does not point to any existing object in the configuration tree, first missing path segment is called '{segment}'"
+ )
+ elif segment in obj:
+ parent = obj
+ obj = obj[segment]
+ else:
+ parent = obj
+ obj = None
+
+ return rwcopy, parent, obj, segment
+
+ @staticmethod
+ def _post(
+ fakeroot: "QueryTree",
+ parent: "QueryTree",
+ obj: Optional["QueryTree"],
+ name: str,
+ update_with: Optional["QueryTree"] = None,
+ ) -> "Tuple[QueryTree, Optional[QueryTree]]":
+ # pylint: disable=protected-access
+ if update_with is None:
+ raise DataParsingError("query invalid: can't request a change via POST and not provide a value")
+ if parent._is_dict():
+ parent._upsert(name, update_with)
+ return fakeroot["root"], None
+ elif parent._is_list():
+ if obj is None:
+ parent._append(update_with)
+ return fakeroot["root"], None
+ else:
+ parent._upsert(name, update_with)
+ return fakeroot["root"], None
+ else:
+ assert False, "this should never happen"
+
+ @staticmethod
+ def _patch(
+ fakeroot: "QueryTree",
+ parent: "QueryTree",
+ obj: Optional["QueryTree"],
+ name: str,
+ update_with: Optional["QueryTree"] = None,
+ ) -> "Tuple[QueryTree, Optional[QueryTree]]":
+ # pylint: disable=protected-access
+ if update_with is None:
+ raise DataParsingError("query invalid: can't request a change via PATCH and not provide a value")
+ if obj is None:
+ raise DataParsingError("query error: can't update non-existent object")
+ else:
+ parent._upsert(name, update_with)
+ return fakeroot["root"], None
+
+ @staticmethod
+ def _put(
+ fakeroot: "QueryTree",
+ parent: "QueryTree",
+ obj: Optional["QueryTree"],
+ name: str,
+ update_with: Optional["QueryTree"] = None,
+ ) -> "Tuple[QueryTree, Optional[QueryTree]]":
+ # pylint: disable=protected-access
+ if update_with is None:
+ raise DataParsingError("query invalid: can't request an insert via PUT and not provide a value")
+ if obj is None:
+ if parent._is_list():
+ parent._append(update_with)
+ return fakeroot["root"], None
+ elif parent._is_dict():
+ parent._upsert(name, update_with)
+ return fakeroot["root"], None
+ else:
+ assert False, "never happens"
+ else:
+ raise DataParsingError("query invalid: can't insert when there is already a value there")
+
+ def query(
+ self, op: Literal["get", "post", "delete", "patch", "put"], path: str, update_with: Optional["QueryTree"] = None
+ ) -> "Tuple[QueryTree, Optional[QueryTree]]":
+ """
+ Implements a modification API in the style of Caddy:
+ https://caddyserver.com/docs/api
+ """
+ # pylint: disable=protected-access
+ fakeroot, parent, obj, name = self._copy_and_find(path)
+
+ # get = return what the path selector picks
+ if op == "get":
+ return fakeroot["root"], obj
+
+ # post = set value at a key, append to lists
+ elif op == "post":
+ return self._post(fakeroot, parent, obj, name, update_with)
+
+ # delete = remove the given key
+ elif op == "delete":
+ parent._delete(name)
+ return fakeroot["root"], None
+
+ # patch = update an existing object
+ elif op == "patch":
+ return self._patch(fakeroot, parent, obj, name, update_with)
+
+ # put = insert and never replace
+ elif op == "put":
+ return self._put(fakeroot, parent, obj, name, update_with)
+
+ else:
+ assert False, "invalid operation"
assert o.o == {"key1": "str1", "key2": "str2"}
-def test_partial_mutations():
- class InnerSchema(BaseSchema):
- size: int = 5
-
- class ConfPreviousSchema(BaseSchema):
- workers: Union[Literal["auto"], int] = 1
- lua_config: Optional[str] = None
- inner: InnerSchema = InnerSchema()
-
- class ConfSchema(BaseSchema):
- _LAYER = ConfPreviousSchema
-
- workers: int
- lua_config: Optional[str]
- inner: InnerSchema
-
- def _workers(self, obj: Any) -> Any:
- if "workers" in obj and obj["workers"] == "auto":
- return 8
- return obj["workers"]
-
- def _validate(self) -> None:
- if self.workers < 0:
- raise ValueError("Number of workers must be non-negative")
-
- yaml = """
- workers: auto
- lua-config: something
- """
-
- d = parse_yaml(yaml)
- o = ConfSchema(d)
- assert o.lua_config == "something"
- assert o.inner.size == 5
- assert o.workers == 8
-
- # replacement of 'lua-config' attribute
- upd = d.update("/lua-config", parse_json('"new_value"'))
- o = ConfSchema(upd)
- assert o.lua_config == "new_value"
- assert o.inner.size == 5
- assert o.workers == 8
-
- # replacement of the whole tree
- o = ConfSchema(d.update("/", parse_json('{"inner": {"size": 55}}')))
- assert o.lua_config is None
- assert o.workers == 1
- assert o.inner.size == 55
-
- # replacement of 'inner' subtree
- o = ConfSchema(d.update("/inner", parse_json('{"size": 33}')))
- assert o.lua_config == "something"
- assert o.workers == 8
- assert o.inner.size == 33
-
- # raise validation DataValidationError
- with raises(DataValidationError):
- o = ConfSchema(d.update("/", parse_json('{"workers": -5}')))
-
-
def test_dash_conversion():
class TestSchema(BaseSchema):
awesome_field: Dict[str, str]
--- /dev/null
+from pyparsing import empty
+
+from knot_resolver_manager.utils.modeling import ParsedTree
+
+
+def test_etag():
+ empty1 = ParsedTree({})
+ empty2 = ParsedTree({})
+
+ assert empty1.etag == empty2.etag
+
+ something1 = ParsedTree({"something": 1})
+ something2 = ParsedTree({"something": 2})
+ assert empty1.etag != something1.etag
+ assert something1.etag != something2.etag
--- /dev/null
+from typing import List, Optional
+
+from pytest import raises
+
+from knot_resolver_manager.utils.modeling.base_schema import BaseSchema
+from knot_resolver_manager.utils.modeling.exceptions import DataValidationError
+from knot_resolver_manager.utils.modeling.parsing import parse_json, parse_yaml
+
+
+class InnerSchema(BaseSchema):
+ size: int = 5
+ lst: Optional[List[int]]
+
+
+class ConfSchema(BaseSchema):
+ workers: int
+ lua_config: Optional[str]
+ inner: InnerSchema = InnerSchema()
+
+ def _validate(self) -> None:
+ super()._validate()
+ if self.workers < 0:
+ raise DataValidationError("ee", "/workers")
+
+
+YAML = """
+workers: 1
+lua-config: something
+"""
+REF = parse_yaml(YAML)
+
+
+def test_patch():
+ o = ConfSchema(REF)
+ assert o.lua_config == "something"
+ assert o.workers == 1
+ assert o.inner.size == 5
+
+ # replacement of 'lua-config' attribute
+ upd, _resp = REF.query("patch", "/lua-config", parse_json('"new_value"'))
+ o = ConfSchema(upd)
+ assert o.lua_config == "new_value"
+ assert o.inner.size == 5
+ assert o.workers == 1
+
+ # replacement of the whole tree
+ upd, _resp = REF.query("patch", "/", parse_json('{"inner": {"size": 55}, "workers": 8}'))
+ o = ConfSchema(upd)
+ assert o.lua_config is None
+ assert o.workers == 8
+ assert o.inner.size == 55
+
+ # raise validation DataValidationError
+ with raises(DataValidationError):
+ upd, _resp = REF.query("patch", "/", parse_json('{"workers": -5}'))
+ o = ConfSchema(upd)
+
+
+def test_put_and_delete():
+ # insert 'inner' subtree
+ upd, _resp = REF.query("put", "/inner", parse_json('{"size": 33}'))
+ o = ConfSchema(upd)
+ assert o.lua_config == "something"
+ assert o.workers == 1
+ assert o.inner.size == 33
+
+ upd, _resp = upd.query("put", "/inner/lst", parse_json("[1,2,3]"))
+ o = ConfSchema(upd)
+ assert tuple(o.inner.lst or []) == tuple([1, 2, 3])
+
+ upd, _resp = upd.query("delete", "/inner/lst/1")
+ o = ConfSchema(upd)
+ assert tuple(o.inner.lst or []) == tuple([1, 3])
+
+ upd, _resp = upd.query("delete", "/inner/lst")
+ o = ConfSchema(upd)
+ assert o.inner.lst is None
projects / thirdparty / knot-resolver.git / commitdiff
