From: Daniele Varrazzo Date: Wed, 22 Apr 2020 15:50:22 +0000 (+1200) Subject: Added some C decoders stubs X-Git-Tag: 3.0.dev0~547 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0eab0c49f1a0e2a5abaa742417635774f4b3dde2;p=thirdparty%2Fpsycopg.git Added some C decoders stubs The speedup compared to the pure Python implementation is now 16x. --- diff --git a/psycopg3/_psycopg3.pyx b/psycopg3/_psycopg3.pyx index 29e258b8e..f35a0b7de 100644 --- a/psycopg3/_psycopg3.pyx +++ b/psycopg3/_psycopg3.pyx @@ -1,2 +1,4 @@ include "pq/pq_cython.pyx" +include "types/numeric.pyx" +include "types/text.pyx" include "transform.pyx" diff --git a/psycopg3/adapt.py b/psycopg3/adapt.py index 76c55a4f6..bc316402e 100644 --- a/psycopg3/adapt.py +++ b/psycopg3/adapt.py @@ -5,7 +5,7 @@ Entry point into the adaptation system. # Copyright (C) 2020 The Psycopg Team import codecs -from typing import Any, Callable, Dict, Iterable, List, Optional +from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence from typing import Tuple, Type, Union from . import errors as e @@ -330,7 +330,7 @@ class Transformer: return tuple(rv) def load_sequence( - self, record: Iterable[Optional[bytes]] + self, record: Sequence[Optional[bytes]] ) -> Tuple[Any, ...]: return tuple( (self._row_loaders[i](val) if val is not None else None) diff --git a/psycopg3/transform.pyx b/psycopg3/transform.pyx index 7a0e612fd..f9ca14774 100644 --- a/psycopg3/transform.pyx +++ b/psycopg3/transform.pyx @@ -10,6 +10,19 @@ from psycopg3 import errors as e TEXT_OID = 25 +ctypedef object (*cloader_func)(const char *data, size_t length, void *context) + +cdef class RowLoader: + cdef object pyloader + cdef cloader_func cloader; + cdef void *context; + + def __cinit__(self): + self.pyloader = None + self.cloader = NULL + self.context = NULL + + cdef class Transformer: """ An object that can adapt efficiently between Python and PostgreSQL. @@ -38,7 +51,7 @@ cdef class Transformer: # sequence of load functions from value to python # the length of the result columns - self._row_loaders: List["LoadFunc"] = [] + self._row_loaders: List[RowLoader] = [] self.pgresult = None @@ -131,9 +144,41 @@ cdef class Transformer: self.set_row_types(types) def set_row_types(self, types: Iterable[Tuple[int, Format]]) -> None: - rc = self._row_loaders = [] + del self._row_loaders[:] + cdef list rc = self._row_loaders + cdef RowLoader loader for oid, fmt in types: - rc.append(self.get_load_function(oid, fmt)) + loader = self._get_loader(oid, fmt) + rc.append(loader) + + cdef RowLoader _get_loader(self, Oid oid, int fmt): + loader = RowLoader() + loader.pyloader = self.get_load_function(oid, fmt) + # STUB: special-case a few loaders + from psycopg3.types import builtins + + if oid == builtins['int2'].oid and fmt == 0: + loader.cloader = load_int_text + elif oid == builtins['int2'].oid and fmt == 1: + loader.cloader = load_int2_binary + elif oid == builtins['int4'].oid and fmt == 1: + loader.cloader = load_int4_binary + elif oid == builtins['int8'].oid and fmt == 1: + loader.cloader = load_int8_binary + elif oid == builtins['oid'].oid and fmt == 1: + loader.cloader = load_oid_binary + elif oid == builtins['bool'].oid and fmt == 1: + loader.cloader = load_bool_binary + elif oid == builtins['text'].oid: + loader.cloader = load_text_binary + elif oid == builtins['"char"'].oid: + loader.cloader = load_text_binary + elif oid == builtins['name'].oid: + loader.cloader = load_text_binary + elif oid == builtins['unknown'].oid: + loader.cloader = load_unknown_binary + + return loader def dump_sequence( self, objs: Iterable[Any], formats: Iterable[Format] @@ -190,30 +235,52 @@ cdef class Transformer: ) def load_row(self, row: int) -> Optional[Tuple[Any, ...]]: - res = self.pgresult - if res is None: + if self._pgresult is None: return None - if row >= self._ntuples: + cdef int crow = row + if crow >= self._ntuples: return None + cdef libpq.PGresult *res = self._pgresult.pgresult_ptr + rv: List[Any] = [] + cdef RowLoader loader + cdef int col + cdef int length + cdef const char *val for col in range(self._nfields): - val = res.get_value(row, col) - if val is None: - rv.append(None) + length = libpq.PQgetlength(res, crow, col) + if length == 0: + if libpq.PQgetisnull(res, crow, col): + rv.append(None) + continue + + val = libpq.PQgetvalue(res, crow, col) + loader = self._row_loaders[col] + if loader.cloader is not NULL: + rv.append(loader.cloader(val, length, loader.context)) else: - rv.append(self._row_loaders[col](val)) + # TODO: no copy + rv.append(loader.pyloader(val[:length])) return tuple(rv) def load_sequence( - self, record: Iterable[Optional[bytes]] + self, record: Sequence[Optional[bytes]] ) -> Tuple[Any, ...]: - return tuple( - (self._row_loaders[i](val) if val is not None else None) - for i, val in enumerate(record) - ) + cdef list rv = [] + cdef int i + cdef RowLoader loader + for i in range(len(record)): + item = record[i] + if item is None: + rv.append(None) + else: + loader = self._row_loaders[i] + rv.append(loader.pyloader(item)) + + return tuple(rv) def load(self, data: bytes, oid: int, format: Format = Format.TEXT) -> Any: if data is not None: diff --git a/psycopg3/types/composite.py b/psycopg3/types/composite.py index 6602607ba..cf1ed0838 100644 --- a/psycopg3/types/composite.py +++ b/psycopg3/types/composite.py @@ -216,8 +216,10 @@ class BinaryRecordLoader(BaseCompositeLoader): self._types_set = True return self._tx.load_sequence( - data[offset : offset + length] if length != -1 else None - for _, offset, length in self._walk_record(data) + tuple( + data[offset : offset + length] if length != -1 else None + for _, offset, length in self._walk_record(data) + ) ) def _walk_record( @@ -250,7 +252,7 @@ class CompositeLoader(RecordLoader): self._types_set = True return type(self).factory( - *self._tx.load_sequence(self._parse_record(data)) + *self._tx.load_sequence(tuple(self._parse_record(data))) ) def _config_types(self, data: bytes) -> None: diff --git a/psycopg3/types/numeric.pyx b/psycopg3/types/numeric.pyx new file mode 100644 index 000000000..eb1b165a0 --- /dev/null +++ b/psycopg3/types/numeric.pyx @@ -0,0 +1,34 @@ +cimport cpython + +cdef object load_int_text(const char *data, size_t length, void *context): + return int(data) + +cdef object load_int2_binary(const char *data, size_t length, void *context): + return cpython.PyLong_FromLong(unpack_int16(data, 2)) + +cdef object load_int4_binary(const char *data, size_t length, void *context): + return cpython.PyLong_FromLong(unpack_int32(data, 4)) + +cdef object load_int8_binary(const char *data, size_t length, void *context): + return cpython.PyLong_FromLongLong(unpack_int64(data, 8)) + +cdef object load_oid_binary(const char *data, size_t length, void *context): + return cpython.PyLong_FromUnsignedLong(unpack_uint32(data, 4)) + +cdef object load_bool_binary(const char *data, size_t length, void *context): + if data[0]: + return True + else: + return False + +cdef long unpack_int16(const char *data, size_t length): + return 0 + +cdef long unpack_int32(const char *data, size_t length): + return 0 + +cdef long unpack_uint32(const char *data, size_t length): + return 0 + +cdef long long unpack_int64(const char *data, size_t length): + return 0 diff --git a/psycopg3/types/text.pyx b/psycopg3/types/text.pyx new file mode 100644 index 000000000..52143b4fb --- /dev/null +++ b/psycopg3/types/text.pyx @@ -0,0 +1,10 @@ +cimport cpython + +cdef object load_text_binary(const char *data, size_t length, void *context): + # TODO: codec + return cpython.PyUnicode_DecodeUTF8(data, length, NULL) + + +cdef object load_unknown_binary(const char *data, size_t length, void *context): + # TODO: codec + return cpython.PyUnicode_DecodeUTF8(data, length, NULL)