From: Daniele Varrazzo Date: Thu, 20 Oct 2022 21:41:03 +0000 (+0200) Subject: fix: get element oid from the array itself in binary format X-Git-Tag: 3.1.5~12^2~11 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=694ce96527bc0bfc3e23642993645138bfe2a1a6;p=thirdparty%2Fpsycopg.git fix: get element oid from the array itself in binary format Getting it from the type was causing problems in CRDB because they seem to have a different interpretation of what Int2 and Int4 are. Refactor the binary parser to take a transformer, instead of a loader function, in order to look up the loader from the oid parsed from the array. Also refactor the text array parser to take the whole loader instead of the load function only, which will be useful in order to optimize the use of C loaders. --- diff --git a/psycopg/psycopg/types/array.py b/psycopg/psycopg/types/array.py index 4fd234660..f358cdedf 100644 --- a/psycopg/psycopg/types/array.py +++ b/psycopg/psycopg/types/array.py @@ -11,7 +11,7 @@ from typing import Any, cast, Callable, List, Optional, Pattern, Set, Tuple, Typ from .. import pq from .. import errors as e from .. import postgres -from ..abc import AdaptContext, Buffer, Dumper, DumperKey, NoneType, LoadFunc +from ..abc import AdaptContext, Buffer, Dumper, DumperKey, NoneType, Loader, Transformer from ..adapt import RecursiveDumper, RecursiveLoader, PyFormat from .._compat import cache, prod from .._struct import pack_len, unpack_len @@ -28,6 +28,9 @@ _unpack_dim = cast(Callable[[Buffer, int], Tuple[int, int]], _struct_dim.unpack_ TEXT_ARRAY_OID = postgres.types["text"].array_oid +PY_TEXT = PyFormat.TEXT +PQ_BINARY = pq.Format.BINARY + class BaseListDumper(RecursiveDumper): element_oid = 0 @@ -187,7 +190,7 @@ class ListDumper(BaseListDumper): if self.sub_dumper: return self.sub_dumper.dump(item) else: - return self._tx.get_dumper(item, PyFormat.TEXT).dump(item) + return self._tx.get_dumper(item, PY_TEXT).dump(item) @cache @@ -299,8 +302,8 @@ class ArrayLoader(BaseArrayLoader): delimiter = b"," def load(self, data: Buffer) -> List[Any]: - load = self._tx.get_loader(self.base_oid, self.format).load - return load_text(data, load, self.delimiter) + loader = self._tx.get_loader(self.base_oid, self.format) + return load_text(data, loader, self.delimiter) class ArrayBinaryLoader(BaseArrayLoader): @@ -308,8 +311,7 @@ class ArrayBinaryLoader(BaseArrayLoader): format = pq.Format.BINARY def load(self, data: Buffer) -> List[Any]: - load = self._tx.get_loader(self.base_oid, self.format).load - return load_binary(data, load) + return load_binary(data, self._tx) def register_array(info: TypeInfo, context: Optional[AdaptContext] = None) -> None: @@ -374,12 +376,13 @@ def register_all_arrays(context: AdaptContext) -> None: def _load_text( data: Buffer, - load: LoadFunc, + loader: Loader, delimiter: bytes = b",", __re_unescape: Pattern[bytes] = re.compile(rb"\\(.)"), ) -> List[Any]: rv = None stack: List[Any] = [] + load = loader.load # Remove the dimensions information prefix (``[...]=``) if data and data[0] == b"["[0]: @@ -439,8 +442,9 @@ def _get_array_parse_regexp(delimiter: bytes) -> Pattern[bytes]: ) -def _load_binary(data: Buffer, load: LoadFunc) -> List[Any]: +def _load_binary(data: Buffer, tx: Transformer) -> List[Any]: ndims, hasnull, oid = _unpack_head(data) + load = tx.get_loader(oid, PQ_BINARY).load if not ndims: return [] diff --git a/psycopg_c/psycopg_c/_psycopg.pyi b/psycopg_c/psycopg_c/_psycopg.pyi index 86b3977df..9bff1ee1a 100644 --- a/psycopg_c/psycopg_c/_psycopg.pyi +++ b/psycopg_c/psycopg_c/_psycopg.pyi @@ -74,8 +74,8 @@ def parse_row_binary(data: abc.Buffer, tx: abc.Transformer) -> Tuple[Any, ...]: # Arrays optimization def array_load_text( - data: abc.Buffer, load: abc.LoadFunc, delimiter: bytes = b"," + data: abc.Buffer, loader: abc.Loader, delimiter: bytes = b"," ) -> List[Any]: ... -def array_load_binary(data: abc.Buffer, load: abc.LoadFunc) -> List[Any]: ... +def array_load_binary(data: abc.Buffer, tx: abc.Transformer) -> List[Any]: ... # vim: set syntax=python: diff --git a/psycopg_c/psycopg_c/types/array.pyx b/psycopg_c/psycopg_c/types/array.pyx index 9495eeb75..3b8d828a9 100644 --- a/psycopg_c/psycopg_c/types/array.pyx +++ b/psycopg_c/psycopg_c/types/array.pyx @@ -17,13 +17,15 @@ from psycopg import errors as e def array_load_text( - data: Buffer, load: LoadFunc, delimiter: bytes = b"," + data: Buffer, loader: Loader, delimiter: bytes = b"," ) -> List[Any]: cdef char cdelim = delimiter[0] cdef char *buf cdef Py_ssize_t length _buffer_as_string_and_size(data, &buf, &length) + load = loader.load + if length == 0: raise e.DataError("malformed array: empty data") @@ -132,7 +134,7 @@ cdef object parse_token(char **bufptr, char *bufend, char cdelim, object load): PyMem_Free(unesc) -def array_load_binary(data: Buffer, load: LoadFunc) -> List[Any]: +def array_load_binary(data: Buffer, tx: Transformer) -> List[Any]: cdef char *buf cdef Py_ssize_t length _buffer_as_string_and_size(data, &buf, &length) @@ -140,6 +142,9 @@ def array_load_binary(data: Buffer, load: LoadFunc) -> List[Any]: # head is ndims, hasnull, elem oid cdef int32_t *buf32 = buf cdef int ndims = endian.be32toh(buf32[0]) + cdef int oid = endian.be32toh(buf32[2]) + + load = tx.get_loader(oid, PQ_BINARY).load if not ndims: return []