From: Daniele Varrazzo Date: Fri, 7 Aug 2020 02:38:26 +0000 (+0100) Subject: All C loaders converted to the PyxLoader class X-Git-Tag: 3.0.dev0~458^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=da73ce23797b9c6b620ee537daa13081ae9678b8;p=thirdparty%2Fpsycopg.git All C loaders converted to the PyxLoader class --- diff --git a/psycopg3_c/psycopg3_c/adapt.pyx b/psycopg3_c/psycopg3_c/adapt.pyx index 6b68b908d..1e628b901 100644 --- a/psycopg3_c/psycopg3_c/adapt.pyx +++ b/psycopg3_c/psycopg3_c/adapt.pyx @@ -27,17 +27,15 @@ logger = logging.getLogger("psycopg3.adapt") # TODO: rename after dropping CLoader below cdef class PyxLoader: cdef impl.Oid oid - cdef impl.PGconn* pgconn_ptr cdef object context + cdef object connection def __init__(self, oid: int, context: "AdaptContext" = None): from psycopg3.adapt import _connection_from_context self.oid = oid self.context = context - connection = _connection_from_context(context) - # TODO - self.pgconn_ptr = NULL + self.connection = _connection_from_context(context) cdef object cload(self, const char *data, size_t length): raise NotImplementedError() @@ -49,32 +47,6 @@ cdef class PyxLoader: return self.cload(data, length) -cdef class CLoader: - cdef object pyloader - cdef cloader_func cloader - cdef get_context_func get_context - - -cloaders = {} - -cdef void register_c_loader( - object pyloader, - cloader_func cloader, - get_context_func get_context = NULL -): - """ - Map a python loader to an optimised C version. - - Whenever the python loader would be used the C version may be chosen in - preference. - """ - cdef CLoader cl = CLoader() - cl.pyloader = pyloader - cl.cloader = cloader - cl.get_context = get_context - cloaders[pyloader] = cl - - def register_builtin_c_loaders(): """ Register all the builtin optimized methods. @@ -83,10 +55,6 @@ def register_builtin_c_loaders(): are registered. """ - if cloaders: - logger.warning("c loaders already registered") - return - logger.debug("registering optimised c loaders") register_numeric_c_loaders() register_text_c_loaders() diff --git a/psycopg3_c/psycopg3_c/transform.pyx b/psycopg3_c/psycopg3_c/transform.pyx index 17cd1e2f5..48682f135 100644 --- a/psycopg3_c/psycopg3_c/transform.pyx +++ b/psycopg3_c/psycopg3_c/transform.pyx @@ -8,10 +8,7 @@ too many temporary Python objects and performing less memory copying. # Copyright (C) 2020 The Psycopg Team -from libc.string cimport memset -from cpython.object cimport PyObject from cpython.ref cimport Py_INCREF -from cpython.mem cimport PyMem_Malloc, PyMem_Free from cpython.tuple cimport PyTuple_New, PyTuple_SET_ITEM import codecs diff --git a/psycopg3_c/psycopg3_c/types/numeric.pyx b/psycopg3_c/psycopg3_c/types/numeric.pyx index 642c98307..b554ad004 100644 --- a/psycopg3_c/psycopg3_c/types/numeric.pyx +++ b/psycopg3_c/psycopg3_c/types/numeric.pyx @@ -12,39 +12,52 @@ from cpython.long cimport ( PyLong_FromLong, PyLong_FromLongLong, PyLong_FromUnsignedLong) -cdef object load_int_text(const char *data, size_t length, void *context): - return int(data) +cdef class TextIntLoader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + return int(data) -cdef object load_int2_binary(const char *data, size_t length, void *context): - return PyLong_FromLong(be16toh((data)[0])) +cdef class BinaryInt2Loader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + return PyLong_FromLong(be16toh((data)[0])) -cdef object load_int4_binary(const char *data, size_t length, void *context): - return PyLong_FromLong(be32toh((data)[0])) +cdef class BinaryInt4Loader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + return PyLong_FromLong(be32toh((data)[0])) -cdef object load_int8_binary(const char *data, size_t length, void *context): - return PyLong_FromLongLong(be64toh((data)[0])) +cdef class BinaryInt8Loader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + return PyLong_FromLongLong(be64toh((data)[0])) -cdef object load_oid_binary(const char *data, size_t length, void *context): - return PyLong_FromUnsignedLong(be32toh((data)[0])) +cdef class BinaryOidLoader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + return PyLong_FromUnsignedLong(be32toh((data)[0])) -cdef object load_bool_binary(const char *data, size_t length, void *context): - if data[0]: - return True - else: - return False +cdef class BinaryBoolLoader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + if data[0]: + return True + else: + return False cdef void register_numeric_c_loaders(): logger.debug("registering optimised numeric c loaders") - from psycopg3.types import numeric - register_c_loader(numeric.load_int, load_int_text) - register_c_loader(numeric.load_int2_binary, load_int2_binary) - register_c_loader(numeric.load_int4_binary, load_int4_binary) - register_c_loader(numeric.load_int8_binary, load_int8_binary) - register_c_loader(numeric.load_oid_binary, load_oid_binary) - register_c_loader(numeric.load_bool_binary, load_bool_binary) + + from psycopg3.adapt import Loader + from psycopg3.types import builtins + + Loader.register(builtins["int2"].oid, TextIntLoader) + Loader.register(builtins["int4"].oid, TextIntLoader) + Loader.register(builtins["int8"].oid, TextIntLoader) + Loader.register(builtins["oid"].oid, TextIntLoader) + + Loader.register_binary(builtins["int2"].oid, BinaryInt2Loader) + Loader.register_binary(builtins["int4"].oid, BinaryInt4Loader) + Loader.register_binary(builtins["int8"].oid, BinaryInt8Loader) + Loader.register_binary(builtins["oid"].oid, BinaryOidLoader) + Loader.register_binary(builtins["bool"].oid, BinaryBoolLoader) diff --git a/psycopg3_c/psycopg3_c/types/text.pyx b/psycopg3_c/psycopg3_c/types/text.pyx index ec600bfe6..188bafe83 100644 --- a/psycopg3_c/psycopg3_c/types/text.pyx +++ b/psycopg3_c/psycopg3_c/types/text.pyx @@ -5,59 +5,52 @@ Cython adapters for textual types. # Copyright (C) 2020 The Psycopg Team from cpython.bytes cimport PyBytes_FromStringAndSize -from cpython.mem cimport PyMem_Malloc -from cpython.object cimport PyObject from cpython.unicode cimport PyUnicode_DecodeUTF8 from psycopg3_c cimport libpq -cdef struct TextContext: - PyObject *pydecoder - int is_utf8 +cdef class StringLoader(PyxLoader): + cdef int is_utf8 + cdef object pydecoder + def __init__(self, oid: int, context: "AdaptContext" = None): + super().__init__(oid, context) -cdef object load_text(const char *data, size_t length, void *context): - cdef TextContext *tcontext = context - if tcontext.is_utf8: - return PyUnicode_DecodeUTF8(data, length, NULL) - - b = PyBytes_FromStringAndSize(data, length) - decoder = (tcontext.pydecoder) - if decoder is not None: - # TODO: check if the refcount is right - return decoder(b)[0] - else: - return b - - -cdef void *get_context_text(object loader): - cdef TextContext *rv = PyMem_Malloc(sizeof(TextContext)) - rv.pydecoder = loader.decode - - if loader.connection is None or loader.connection.encoding == "UTF8": - rv.is_utf8 = 1 - else: - rv.is_utf8 = 0 - - return rv + self.is_utf8 = 0 + self.pydecoder = None + conn = self.connection + if conn is not None: + if conn.encoding == "UTF8": + self.is_utf8 = 1 + elif conn.encoding != "SQL_ASCII": + self.pydecoder = conn.codec.decode + else: + self.pydecoder = codecs.lookup("utf8").decode + cdef object cload(self, const char *data, size_t length): + if self.is_utf8: + return PyUnicode_DecodeUTF8(data, length, NULL) -cdef object load_bytea_text(const char *data, size_t length, void *context): - cdef size_t len_out - cdef unsigned char *out = libpq.PQunescapeBytea( - data, &len_out) - if out is NULL: - raise MemoryError( - f"couldn't allocate for unescape_bytea of {len(data)} bytes" - ) + b = PyBytes_FromStringAndSize(data, length) + if self.pydecoder is not None: + return self.pydecoder(b)[0] + else: + return b - rv = out[:len_out] - libpq.PQfreemem(out) - return rv +cdef class TextByteaLoader(PyxLoader): + cdef object cload(self, const char *data, size_t length): + cdef size_t len_out + cdef unsigned char *out = libpq.PQunescapeBytea( + data, &len_out) + if out is NULL: + raise MemoryError( + f"couldn't allocate for unescape_bytea of {len(data)} bytes" + ) -cdef object load_bytea_binary(const char *data, size_t length, void *context): - return data[:length] + rv = out[:len_out] + libpq.PQfreemem(out) + return rv cdef class BinaryByteaLoader(PyxLoader): @@ -67,6 +60,15 @@ cdef class BinaryByteaLoader(PyxLoader): cdef void register_text_c_loaders(): logger.debug("registering optimised text c loaders") + from psycopg3.adapt import Loader from psycopg3.types import builtins + + Loader.register(0, StringLoader) # INVALID_OID + Loader.register(builtins["text"].oid, StringLoader) + Loader.register_binary(builtins["text"].oid, StringLoader) + Loader.register(builtins["varchar"].oid, StringLoader) + Loader.register_binary(builtins["varchar"].oid, StringLoader) + + Loader.register(builtins['bytea'].oid, TextByteaLoader) Loader.register_binary(builtins['bytea'].oid, BinaryByteaLoader)