Added some C decoders stubs

The speedup compared to the pure Python implementation is now 16x.

diff --git a/psycopg3/_psycopg3.pyx b/psycopg3/_psycopg3.pyx
index 29e258b8efa63f0e19112f6568d6fca6aa9ab5f3..f35a0b7de8b24437177a5babc84b61843a3d3fc1 100644 (file)
--- a/psycopg3/_psycopg3.pyx
+++ b/psycopg3/_psycopg3.pyx
@@ -1,2 +1,4 @@
 include "pq/pq_cython.pyx"
+include "types/numeric.pyx"
+include "types/text.pyx"
 include "transform.pyx"

diff --git a/psycopg3/adapt.py b/psycopg3/adapt.py
index 76c55a4f6167ff9f891374bf74c8406e8818bd3d..bc316402eb04ea091986c6bee3e6844324c2c990 100644 (file)
--- a/psycopg3/adapt.py
+++ b/psycopg3/adapt.py
@@ -5,7 +5,7 @@ Entry point into the adaptation system.
 # Copyright (C) 2020 The Psycopg Team
 
 import codecs
-from typing import Any, Callable, Dict, Iterable, List, Optional
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence
 from typing import Tuple, Type, Union
 
 from . import errors as e
@@ -330,7 +330,7 @@ class Transformer:
         return tuple(rv)
 
     def load_sequence(
-        self, record: Iterable[Optional[bytes]]
+        self, record: Sequence[Optional[bytes]]
     ) -> Tuple[Any, ...]:
         return tuple(
             (self._row_loaders[i](val) if val is not None else None)

diff --git a/psycopg3/transform.pyx b/psycopg3/transform.pyx
index 7a0e612fd35f2922404f2899daccdc57fc7f7f40..f9ca147749445cfd76591bd2696a115b67aeecf9 100644 (file)
--- a/psycopg3/transform.pyx
+++ b/psycopg3/transform.pyx
@@ -10,6 +10,19 @@ from psycopg3 import errors as e
 TEXT_OID = 25
 
 
+ctypedef object (*cloader_func)(const char *data, size_t length, void *context)
+
+cdef class RowLoader:
+    cdef object pyloader
+    cdef cloader_func cloader;
+    cdef void *context;
+
+    def __cinit__(self):
+        self.pyloader = None
+        self.cloader = NULL
+        self.context = NULL
+
+
 cdef class Transformer:
     """
     An object that can adapt efficiently between Python and PostgreSQL.
@@ -38,7 +51,7 @@ cdef class Transformer:
 
         # sequence of load functions from value to python
         # the length of the result columns
-        self._row_loaders: List["LoadFunc"] = []
+        self._row_loaders: List[RowLoader] = []
 
         self.pgresult = None
 
@@ -131,9 +144,41 @@ cdef class Transformer:
         self.set_row_types(types)
 
     def set_row_types(self, types: Iterable[Tuple[int, Format]]) -> None:
-        rc = self._row_loaders = []
+        del self._row_loaders[:]
+        cdef list rc = self._row_loaders
+        cdef RowLoader loader
         for oid, fmt in types:
-            rc.append(self.get_load_function(oid, fmt))
+            loader = self._get_loader(oid, fmt)
+            rc.append(loader)
+
+    cdef RowLoader _get_loader(self, Oid oid, int fmt):
+        loader = RowLoader()
+        loader.pyloader = self.get_load_function(oid, fmt)
+        # STUB: special-case a few loaders
+        from psycopg3.types import builtins
+
+        if oid == builtins['int2'].oid and fmt == 0:
+            loader.cloader = load_int_text
+        elif oid == builtins['int2'].oid and fmt == 1:
+            loader.cloader = load_int2_binary
+        elif oid == builtins['int4'].oid and fmt == 1:
+            loader.cloader = load_int4_binary
+        elif oid == builtins['int8'].oid and fmt == 1:
+            loader.cloader = load_int8_binary
+        elif oid == builtins['oid'].oid and fmt == 1:
+            loader.cloader = load_oid_binary
+        elif oid == builtins['bool'].oid and fmt == 1:
+            loader.cloader = load_bool_binary
+        elif oid == builtins['text'].oid:
+            loader.cloader = load_text_binary
+        elif oid == builtins['"char"'].oid:
+            loader.cloader = load_text_binary
+        elif oid == builtins['name'].oid:
+            loader.cloader = load_text_binary
+        elif oid == builtins['unknown'].oid:
+            loader.cloader = load_unknown_binary
+
+        return loader
 
     def dump_sequence(
         self, objs: Iterable[Any], formats: Iterable[Format]
@@ -190,30 +235,52 @@ cdef class Transformer:
         )
 
     def load_row(self, row: int) -> Optional[Tuple[Any, ...]]:
-        res = self.pgresult
-        if res is None:
+        if self._pgresult is None:
             return None
 
-        if row >= self._ntuples:
+        cdef int crow = row
+        if crow >= self._ntuples:
             return None
 
+        cdef libpq.PGresult *res = self._pgresult.pgresult_ptr
+
         rv: List[Any] = []
+        cdef RowLoader loader
+        cdef int col
+        cdef int length
+        cdef const char *val
         for col in range(self._nfields):
-            val = res.get_value(row, col)
-            if val is None:
-                rv.append(None)
+            length = libpq.PQgetlength(res, crow, col)
+            if length == 0:
+                if libpq.PQgetisnull(res, crow, col):
+                    rv.append(None)
+                    continue
+
+            val = libpq.PQgetvalue(res, crow, col)
+            loader = self._row_loaders[col]
+            if loader.cloader is not NULL:
+                rv.append(loader.cloader(val, length, loader.context))
             else:
-                rv.append(self._row_loaders[col](val))
+                # TODO: no copy
+                rv.append(loader.pyloader(val[:length]))
 
         return tuple(rv)
 
     def load_sequence(
-        self, record: Iterable[Optional[bytes]]
+        self, record: Sequence[Optional[bytes]]
     ) -> Tuple[Any, ...]:
-        return tuple(
-            (self._row_loaders[i](val) if val is not None else None)
-            for i, val in enumerate(record)
-        )
+        cdef list rv = []
+        cdef int i
+        cdef RowLoader loader
+        for i in range(len(record)):
+            item = record[i]
+            if item is None:
+                rv.append(None)
+            else:
+                loader = self._row_loaders[i]
+                rv.append(loader.pyloader(item))
+
+        return tuple(rv)
 
     def load(self, data: bytes, oid: int, format: Format = Format.TEXT) -> Any:
         if data is not None:

diff --git a/psycopg3/types/composite.py b/psycopg3/types/composite.py
index 6602607ba84dfb6dc31e9c4075ab6f73d1abaeed..cf1ed0838ed9f8ebeef6fedba7ec441a8b6c0936 100644 (file)
--- a/psycopg3/types/composite.py
+++ b/psycopg3/types/composite.py
@@ -216,8 +216,10 @@ class BinaryRecordLoader(BaseCompositeLoader):
             self._types_set = True
 
         return self._tx.load_sequence(
-            data[offset : offset + length] if length != -1 else None
-            for _, offset, length in self._walk_record(data)
+            tuple(
+                data[offset : offset + length] if length != -1 else None
+                for _, offset, length in self._walk_record(data)
+            )
         )
 
     def _walk_record(
@@ -250,7 +252,7 @@ class CompositeLoader(RecordLoader):
             self._types_set = True
 
         return type(self).factory(
-            *self._tx.load_sequence(self._parse_record(data))
+            *self._tx.load_sequence(tuple(self._parse_record(data)))
         )
 
     def _config_types(self, data: bytes) -> None:

diff --git a/psycopg3/types/numeric.pyx b/psycopg3/types/numeric.pyx
new file mode 100644 (file)
index 0000000..eb1b165
--- /dev/null
+++ b/psycopg3/types/numeric.pyx
@@ -0,0 +1,34 @@
+cimport cpython
+
+cdef object load_int_text(const char *data, size_t length, void *context):
+    return int(data)
+
+cdef object load_int2_binary(const char *data, size_t length, void *context):
+    return cpython.PyLong_FromLong(unpack_int16(data, 2))
+
+cdef object load_int4_binary(const char *data, size_t length, void *context):
+    return cpython.PyLong_FromLong(unpack_int32(data, 4))
+
+cdef object load_int8_binary(const char *data, size_t length, void *context):
+    return cpython.PyLong_FromLongLong(unpack_int64(data, 8))
+
+cdef object load_oid_binary(const char *data, size_t length, void *context):
+    return cpython.PyLong_FromUnsignedLong(unpack_uint32(data, 4))
+
+cdef object load_bool_binary(const char *data, size_t length, void *context):
+    if data[0]:
+        return True
+    else:
+        return False
+
+cdef long unpack_int16(const char *data, size_t length):
+    return 0
+
+cdef long unpack_int32(const char *data, size_t length):
+    return 0
+
+cdef long unpack_uint32(const char *data, size_t length):
+    return 0
+
+cdef long long unpack_int64(const char *data, size_t length):
+    return 0

diff --git a/psycopg3/types/text.pyx b/psycopg3/types/text.pyx
new file mode 100644 (file)
index 0000000..52143b4
--- /dev/null
+++ b/psycopg3/types/text.pyx
@@ -0,0 +1,10 @@
+cimport cpython
+
+cdef object load_text_binary(const char *data, size_t length, void *context):
+    # TODO: codec
+    return cpython.PyUnicode_DecodeUTF8(data, length, NULL)
+
+
+cdef object load_unknown_binary(const char *data, size_t length, void *context):
+    # TODO: codec
+    return cpython.PyUnicode_DecodeUTF8(data, length, NULL)