From: Daniele Varrazzo Date: Sat, 26 Dec 2020 22:12:02 +0000 (+0100) Subject: Some C optimisation of the Transformer object X-Git-Tag: 3.0.dev0~237 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2e8b9674c2ab6d80fa8ce1be3f2ef5d20982af36;p=thirdparty%2Fpsycopg.git Some C optimisation of the Transformer object Don't use a list of 2-tuples to set types, use two lists of scalars. --- diff --git a/psycopg3/psycopg3/_transform.py b/psycopg3/psycopg3/_transform.py index 6e1ce6f6f..385ad8e81 100644 --- a/psycopg3/psycopg3/_transform.py +++ b/psycopg3/psycopg3/_transform.py @@ -4,7 +4,7 @@ Helper object to transform values between Python and PostgreSQL # Copyright (C) 2020 The Psycopg Team -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple from typing import TYPE_CHECKING from . import errors as e @@ -84,10 +84,14 @@ class Transformer(AdaptContext): fmt = result.fformat(i) rc.append(self.get_loader(oid, fmt).load) - def set_row_types(self, types: Iterable[Tuple[int, Format]]) -> None: - rc = self._row_loaders = [] - for oid, fmt in types: - rc.append(self.get_loader(oid, fmt).load) + def set_row_types( + self, types: Sequence[int], formats: Sequence[Format] + ) -> None: + rc: List[LoadFunc] = [None] * len(types) # type: ignore[list-item] + for i in range(len(rc)): + rc[i] = self.get_loader(types[i], formats[i]).load + + self._row_loaders = rc def get_dumper(self, obj: Any, format: Format) -> "Dumper": # Fast path: return a Dumper class already instantiated from the same type diff --git a/psycopg3/psycopg3/proto.py b/psycopg3/psycopg3/proto.py index 7be4cca3a..29a29cd5e 100644 --- a/psycopg3/psycopg3/proto.py +++ b/psycopg3/psycopg3/proto.py @@ -90,7 +90,9 @@ class Transformer(Protocol): def pgresult(self, result: Optional[pq.proto.PGresult]) -> None: ... - def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None: + def set_row_types( + self, types: Sequence[int], formats: Sequence[Format] + ) -> None: ... def get_dumper(self, obj: Any, format: Format) -> "Dumper": diff --git a/psycopg3/psycopg3/types/composite.py b/psycopg3/psycopg3/types/composite.py index ebebf649e..53abd2aa4 100644 --- a/psycopg3/psycopg3/types/composite.py +++ b/psycopg3/psycopg3/types/composite.py @@ -86,7 +86,7 @@ class CompositeInfo(TypeInfo): (CompositeLoader,), { "factory": factory, - "fields_types": tuple(f.type_oid for f in self.fields), + "fields_types": [f.type_oid for f in self.fields], }, ) loader.register(self.oid, context=context, format=Format.TEXT) @@ -270,14 +270,13 @@ class RecordBinaryLoader(BaseCompositeLoader): i += (8 + length) if length > 0 else 8 def _config_types(self, data: bytes) -> None: - self._tx.set_row_types( - [(oid, Format.BINARY) for oid, _, _ in self._walk_record(data)] - ) + oids = [r[0] for r in self._walk_record(data)] + self._tx.set_row_types(oids, [Format.BINARY] * len(oids)) class CompositeLoader(RecordLoader): factory: Callable[..., Any] - fields_types: Tuple[int, ...] + fields_types: List[int] _types_set = False def load(self, data: bytes) -> Any: @@ -294,7 +293,7 @@ class CompositeLoader(RecordLoader): def _config_types(self, data: bytes) -> None: self._tx.set_row_types( - [(oid, Format.TEXT) for oid in self.fields_types] + self.fields_types, [Format.TEXT] * len(self.fields_types) ) diff --git a/psycopg3_c/psycopg3_c/_psycopg3.pyi b/psycopg3_c/psycopg3_c/_psycopg3.pyi index 87bf30103..146436bca 100644 --- a/psycopg3_c/psycopg3_c/_psycopg3.pyi +++ b/psycopg3_c/psycopg3_c/_psycopg3.pyi @@ -25,7 +25,9 @@ class Transformer(AdaptContext): def pgresult(self) -> Optional[PGresult]: ... @pgresult.setter def pgresult(self, result: Optional[PGresult]) -> None: ... - def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None: ... + def set_row_types( + self, types: Sequence[int], formats: Sequence[Format] + ) -> None: ... def get_dumper(self, obj: Any, format: Format) -> Dumper: ... def load_row(self, row: int) -> Optional[Tuple[Any, ...]]: ... def load_sequence( diff --git a/psycopg3_c/psycopg3_c/transform.pyx b/psycopg3_c/psycopg3_c/transform.pyx index 4d2127bac..7671dc926 100644 --- a/psycopg3_c/psycopg3_c/transform.pyx +++ b/psycopg3_c/psycopg3_c/transform.pyx @@ -53,7 +53,9 @@ cdef class Transformer: cdef readonly object connection cdef readonly object adapters - cdef dict _dumpers_cache, _loaders_cache + cdef dict _dumpers_cache + cdef dict _text_loaders + cdef dict _binary_loaders cdef PGresult _pgresult cdef int _nfields, _ntuples cdef list _row_loaders @@ -70,8 +72,9 @@ cdef class Transformer: # mapping class, fmt -> Dumper instance self._dumpers_cache: Dict[Tuple[type, Format], "Dumper"] = {} - # mapping oid, fmt -> Loader instance - self._loaders_cache: Dict[Tuple[int, Format], "Loader"] = {} + # mapping oid -> Loader instance (text, binary) + self._text_loaders = {} + self._binary_loaders = {} self.pgresult = None self._row_loaders = [] @@ -93,29 +96,44 @@ cdef class Transformer: self._ntuples = libpq.PQntuples(res) cdef int i - types = [ - (libpq.PQftype(res, i), libpq.PQfformat(res, i)) - for i in range(self._nfields)] - self.set_row_types(types) - - def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None: - del self._row_loaders[:] - - cdef int i = 0 - cdef dict seen = {} - for oid_fmt in types: - if oid_fmt not in seen: - self._row_loaders.append( - self._get_row_loader(oid_fmt[0], oid_fmt[1])) - seen[oid_fmt] = i + cdef list types = [None] * self._nfields + cdef list formats = [None] * self._nfields + for i in range(self._nfields): + types[i] = libpq.PQftype(res, i) + formats[i] = libpq.PQfformat(res, i) + self.set_row_types(types, formats) + + def set_row_types(self, types: Sequence[int], formats: Sequence[Format]) -> None: + self._c_set_row_types(types, formats) + + cdef void _c_set_row_types(self, list types, list formats): + cdef dict text_loaders = {} + cdef dict binary_loaders = {} + cdef list loaders = [None] * len(types) + cdef libpq.Oid oid + cdef int fmt + cdef int i + for i in range(len(types)): + oid = types[i] + fmt = formats[i] + if fmt == 0: + if oid in text_loaders: + loaders[i] = text_loaders[oid] + else: + loaders[i] = text_loaders[oid] \ + = self._get_row_loader(oid, fmt) else: - self._row_loaders.append(self._row_loaders[seen[oid_fmt]]) + if oid in binary_loaders: + loaders[i] = binary_loaders[oid] + else: + loaders[i] = binary_loaders[oid] \ + = self._get_row_loader(oid, fmt) - i += 1 + self._row_loaders = loaders cdef RowLoader _get_row_loader(self, libpq.Oid oid, int fmt): cdef RowLoader row_loader = RowLoader() - loader = self.get_loader(oid, fmt) + loader = self._c_get_loader(oid, fmt) row_loader.pyloader = loader.load if isinstance(loader, CLoader): @@ -206,31 +224,39 @@ cdef class Transformer: return rv - def load_sequence( - self, record: Sequence[Optional[bytes]] - ) -> Tuple[Any, ...]: - cdef list rv = [] - cdef int i + def load_sequence(self, record: Sequence[Optional[bytes]]) -> Tuple[Any, ...]: + cdef int length = len(record) + rv = PyTuple_New(length) cdef RowLoader loader - for i in range(len(record)): + + cdef int i + for i in range(length): item = record[i] if item is None: - rv.append(None) + pyval = None else: loader = self._row_loaders[i] - rv.append(loader.pyloader(item)) + pyval = loader.pyloader(item) + Py_INCREF(pyval) + PyTuple_SET_ITEM(rv, i, pyval) - return tuple(rv) + return rv def get_loader(self, oid: int, format: Format) -> "Loader": - key = (oid, format) - try: - return self._loaders_cache[key] - except KeyError: - pass + return self._c_get_loader(oid, format) + + cdef object _c_get_loader(self, libpq.Oid oid, int format): + cdef dict cache + if format == 0: + cache = self._text_loaders + else: + cache = self._binary_loaders + + if oid in cache: + return cache[oid] - loader_cls = self.adapters._loaders.get(key) + loader_cls = self.adapters._loaders.get((oid, format)) if loader_cls is None: loader_cls = self.adapters._loaders[oids.INVALID_OID, format] - loader = self._loaders_cache[key] = loader_cls(oid, self) + loader = cache[oid] = loader_cls(oid, self) return loader