Some C optimisation of the Transformer object

Don't use a list of 2-tuples to set types, use two lists of scalars.

diff --git a/psycopg3/psycopg3/_transform.py b/psycopg3/psycopg3/_transform.py
index 6e1ce6f6fd004f493323ddd097395c80bec23247..385ad8e81e7177262787751c267d7876100666ad 100644 (file)
--- a/psycopg3/psycopg3/_transform.py
+++ b/psycopg3/psycopg3/_transform.py
@@ -4,7 +4,7 @@ Helper object to transform values between Python and PostgreSQL
 
 # Copyright (C) 2020 The Psycopg Team
 
-from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+from typing import Any, Dict, List, Optional, Sequence, Tuple
 from typing import TYPE_CHECKING
 
 from . import errors as e
@@ -84,10 +84,14 @@ class Transformer(AdaptContext):
             fmt = result.fformat(i)
             rc.append(self.get_loader(oid, fmt).load)
 
-    def set_row_types(self, types: Iterable[Tuple[int, Format]]) -> None:
-        rc = self._row_loaders = []
-        for oid, fmt in types:
-            rc.append(self.get_loader(oid, fmt).load)
+    def set_row_types(
+        self, types: Sequence[int], formats: Sequence[Format]
+    ) -> None:
+        rc: List[LoadFunc] = [None] * len(types)  # type: ignore[list-item]
+        for i in range(len(rc)):
+            rc[i] = self.get_loader(types[i], formats[i]).load
+
+        self._row_loaders = rc
 
     def get_dumper(self, obj: Any, format: Format) -> "Dumper":
         # Fast path: return a Dumper class already instantiated from the same type

diff --git a/psycopg3/psycopg3/proto.py b/psycopg3/psycopg3/proto.py
index 7be4cca3aae262b2556ae95934e4f514b8b03f89..29a29cd5e69dadc164d08703bf1d079cf2fb65c1 100644 (file)
--- a/psycopg3/psycopg3/proto.py
+++ b/psycopg3/psycopg3/proto.py
@@ -90,7 +90,9 @@ class Transformer(Protocol):
     def pgresult(self, result: Optional[pq.proto.PGresult]) -> None:
         ...
 
-    def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None:
+    def set_row_types(
+        self, types: Sequence[int], formats: Sequence[Format]
+    ) -> None:
         ...
 
     def get_dumper(self, obj: Any, format: Format) -> "Dumper":

diff --git a/psycopg3/psycopg3/types/composite.py b/psycopg3/psycopg3/types/composite.py
index ebebf649e21952eb5a0ee5a6815d91750aef3a19..53abd2aa46696f26315755195e88a8b9777f15e8 100644 (file)
--- a/psycopg3/psycopg3/types/composite.py
+++ b/psycopg3/psycopg3/types/composite.py
@@ -86,7 +86,7 @@ class CompositeInfo(TypeInfo):
             (CompositeLoader,),
             {
                 "factory": factory,
-                "fields_types": tuple(f.type_oid for f in self.fields),
+                "fields_types": [f.type_oid for f in self.fields],
             },
         )
         loader.register(self.oid, context=context, format=Format.TEXT)
@@ -270,14 +270,13 @@ class RecordBinaryLoader(BaseCompositeLoader):
             i += (8 + length) if length > 0 else 8
 
     def _config_types(self, data: bytes) -> None:
-        self._tx.set_row_types(
-            [(oid, Format.BINARY) for oid, _, _ in self._walk_record(data)]
-        )
+        oids = [r[0] for r in self._walk_record(data)]
+        self._tx.set_row_types(oids, [Format.BINARY] * len(oids))
 
 
 class CompositeLoader(RecordLoader):
     factory: Callable[..., Any]
-    fields_types: Tuple[int, ...]
+    fields_types: List[int]
     _types_set = False
 
     def load(self, data: bytes) -> Any:
@@ -294,7 +293,7 @@ class CompositeLoader(RecordLoader):
 
     def _config_types(self, data: bytes) -> None:
         self._tx.set_row_types(
-            [(oid, Format.TEXT) for oid in self.fields_types]
+            self.fields_types, [Format.TEXT] * len(self.fields_types)
         )
 
 

diff --git a/psycopg3_c/psycopg3_c/_psycopg3.pyi b/psycopg3_c/psycopg3_c/_psycopg3.pyi
index 87bf301034a868803a62b85fa0b4968e031bc649..146436bca6e28657992b7512fc22678653e53ea8 100644 (file)
--- a/psycopg3_c/psycopg3_c/_psycopg3.pyi
+++ b/psycopg3_c/psycopg3_c/_psycopg3.pyi
@@ -25,7 +25,9 @@ class Transformer(AdaptContext):
     def pgresult(self) -> Optional[PGresult]: ...
     @pgresult.setter
     def pgresult(self, result: Optional[PGresult]) -> None: ...
-    def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None: ...
+    def set_row_types(
+        self, types: Sequence[int], formats: Sequence[Format]
+    ) -> None: ...
     def get_dumper(self, obj: Any, format: Format) -> Dumper: ...
     def load_row(self, row: int) -> Optional[Tuple[Any, ...]]: ...
     def load_sequence(

diff --git a/psycopg3_c/psycopg3_c/transform.pyx b/psycopg3_c/psycopg3_c/transform.pyx
index 4d2127bacea1479c37d71402f8e9a448a54141ee..7671dc926b709bee5b9d31a0084ad691c98cb744 100644 (file)
--- a/psycopg3_c/psycopg3_c/transform.pyx
+++ b/psycopg3_c/psycopg3_c/transform.pyx
@@ -53,7 +53,9 @@ cdef class Transformer:
 
     cdef readonly object connection
     cdef readonly object adapters
-    cdef dict _dumpers_cache, _loaders_cache
+    cdef dict _dumpers_cache
+    cdef dict _text_loaders
+    cdef dict _binary_loaders
     cdef PGresult _pgresult
     cdef int _nfields, _ntuples
     cdef list _row_loaders
@@ -70,8 +72,9 @@ cdef class Transformer:
         # mapping class, fmt -> Dumper instance
         self._dumpers_cache: Dict[Tuple[type, Format], "Dumper"] = {}
 
-        # mapping oid, fmt -> Loader instance
-        self._loaders_cache: Dict[Tuple[int, Format], "Loader"] = {}
+        # mapping oid -> Loader instance (text, binary)
+        self._text_loaders = {}
+        self._binary_loaders = {}
 
         self.pgresult = None
         self._row_loaders = []
@@ -93,29 +96,44 @@ cdef class Transformer:
         self._ntuples = libpq.PQntuples(res)
 
         cdef int i
-        types = [
-            (libpq.PQftype(res, i), libpq.PQfformat(res, i))
-            for i in range(self._nfields)]
-        self.set_row_types(types)
-
-    def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None:
-        del self._row_loaders[:]
-
-        cdef int i = 0
-        cdef dict seen = {}
-        for oid_fmt in types:
-            if oid_fmt not in seen:
-                self._row_loaders.append(
-                    self._get_row_loader(oid_fmt[0], oid_fmt[1]))
-                seen[oid_fmt] = i
+        cdef list types = [None] * self._nfields
+        cdef list formats = [None] * self._nfields
+        for i in range(self._nfields):
+            types[i] = libpq.PQftype(res, i)
+            formats[i] = libpq.PQfformat(res, i)
+        self.set_row_types(types, formats)
+
+    def set_row_types(self, types: Sequence[int], formats: Sequence[Format]) -> None:
+        self._c_set_row_types(types, formats)
+
+    cdef void _c_set_row_types(self, list types, list formats):
+        cdef dict text_loaders = {}
+        cdef dict binary_loaders = {}
+        cdef list loaders = [None] * len(types)
+        cdef libpq.Oid oid
+        cdef int fmt
+        cdef int i
+        for i in range(len(types)):
+            oid = types[i]
+            fmt = formats[i]
+            if fmt == 0:
+                if oid in text_loaders:
+                    loaders[i] = text_loaders[oid]
+                else:
+                    loaders[i] = text_loaders[oid] \
+                        = self._get_row_loader(oid, fmt)
             else:
-                self._row_loaders.append(self._row_loaders[seen[oid_fmt]])
+                if oid in binary_loaders:
+                    loaders[i] = binary_loaders[oid]
+                else:
+                    loaders[i] = binary_loaders[oid] \
+                        = self._get_row_loader(oid, fmt)
 
-            i += 1
+        self._row_loaders = loaders
 
     cdef RowLoader _get_row_loader(self, libpq.Oid oid, int fmt):
         cdef RowLoader row_loader = RowLoader()
-        loader = self.get_loader(oid, fmt)
+        loader = self._c_get_loader(oid, fmt)
         row_loader.pyloader = loader.load
 
         if isinstance(loader, CLoader):
@@ -206,31 +224,39 @@ cdef class Transformer:
 
         return rv
 
-    def load_sequence(
-        self, record: Sequence[Optional[bytes]]
-    ) -> Tuple[Any, ...]:
-        cdef list rv = []
-        cdef int i
+    def load_sequence(self, record: Sequence[Optional[bytes]]) -> Tuple[Any, ...]:
+        cdef int length = len(record)
+        rv = PyTuple_New(length)
         cdef RowLoader loader
-        for i in range(len(record)):
+
+        cdef int i
+        for i in range(length):
             item = record[i]
             if item is None:
-                rv.append(None)
+                pyval = None
             else:
                 loader = self._row_loaders[i]
-                rv.append(loader.pyloader(item))
+                pyval = loader.pyloader(item)
+            Py_INCREF(pyval)
+            PyTuple_SET_ITEM(rv, i, pyval)
 
-        return tuple(rv)
+        return rv
 
     def get_loader(self, oid: int, format: Format) -> "Loader":
-        key = (oid, format)
-        try:
-            return self._loaders_cache[key]
-        except KeyError:
-            pass
+        return self._c_get_loader(oid, format)
+
+    cdef object _c_get_loader(self, libpq.Oid oid, int format):
+        cdef dict cache
+        if format == 0:
+            cache = self._text_loaders
+        else:
+            cache = self._binary_loaders
+
+        if oid in cache:
+            return cache[oid]
 
-        loader_cls = self.adapters._loaders.get(key)
+        loader_cls = self.adapters._loaders.get((oid, format))
         if loader_cls is None:
             loader_cls = self.adapters._loaders[oids.INVALID_OID, format]
-        loader = self._loaders_cache[key] = loader_cls(oid, self)
+        loader = cache[oid] = loader_cls(oid, self)
         return loader