From: Daniele Varrazzo Date: Fri, 7 Aug 2020 01:52:14 +0000 (+0100) Subject: Use a C fast path to use the Loader.cload where available X-Git-Tag: 3.0.dev0~458^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ec9a9844fad3ce835463f979024530ec441f81be;p=thirdparty%2Fpsycopg.git Use a C fast path to use the Loader.cload where available --- diff --git a/psycopg3_c/psycopg3_c/transform.pyx b/psycopg3_c/psycopg3_c/transform.pyx index 0e3b0eb0a..17cd1e2f5 100644 --- a/psycopg3_c/psycopg3_c/transform.pyx +++ b/psycopg3_c/psycopg3_c/transform.pyx @@ -26,11 +26,9 @@ from psycopg3.pq.enums import Format TEXT_OID = 25 -cdef struct RowLoader: - PyObject *pyloader # borrowed - cloader_func cloader - void *context - int own_context +cdef class RowLoader: + cdef object pyloader + cdef PyxLoader pyxloader cdef class Transformer: @@ -43,14 +41,12 @@ cdef class Transformer: """ cdef list _dumpers_maps, _loaders_maps - cdef dict _dumpers, _loaders, _dumpers_cache, _load_funcs + cdef dict _dumpers, _loaders, _dumpers_cache, _loaders_cache, _load_funcs cdef object _connection, _codec cdef PGresult _pgresult cdef int _nfields, _ntuples - cdef int _nloaders - cdef RowLoader *_row_loaders - cdef list _pyloaders # only used to keep a reference + cdef list _row_loaders def __cinit__(self, context: "AdaptContext" = None): self._dumpers_maps: List["DumpersMap"] = [] @@ -60,26 +56,14 @@ cdef class Transformer: # mapping class, fmt -> Dumper instance self._dumpers_cache: Dict[Tuple[type, Format], "Dumper"] = {} + # mapping oid, fmt -> Loader instance + self._loaders_cache: Dict[Tuple[int, Format], "Loader"] = {} + # mapping oid, fmt -> load function self._load_funcs: Dict[Tuple[int, Format], "LoadFunc"] = {} - self._nloaders = 0 - self._row_loaders = NULL - self._pyloaders = [] - self.pgresult = None - - def __dealloc__(self): - self._clear_row_loaders() - - cdef _clear_row_loaders(self): - cdef int i - for i in range(self._nloaders): - if self._row_loaders[i].own_context: - PyMem_Free(self._row_loaders[i].context) - PyMem_Free(self._row_loaders) - self._row_loaders = NULL - self._nloaders = 0 + self._row_loaders = [] def _setup_context(self, context: "AdaptContext") -> None: from psycopg3.adapt import Dumper, Loader @@ -170,52 +154,31 @@ cdef class Transformer: self.set_row_types(types) def set_row_types(self, types: Sequence[Tuple[int, Format]]) -> None: - cdef int ntypes = len(types) - self._clear_row_loaders() - self._row_loaders = PyMem_Malloc(ntypes * sizeof(RowLoader)) - memset(self._row_loaders, 0, ntypes * sizeof(RowLoader)) - self._nloaders = ntypes - self._pyloaders = [] - - cdef int i = 0, j + del self._row_loaders[:] + + cdef int i = 0 cdef dict seen = {} for oid_fmt in types: if oid_fmt not in seen: - loader = self._set_loader(i, oid_fmt[0], oid_fmt[1]) + self._row_loaders.append( + self._get_row_loader(oid_fmt[0], oid_fmt[1])) seen[oid_fmt] = i else: - self._copy_loader(seen[oid_fmt], i) + self._row_loaders.append(self._row_loaders[seen[oid_fmt]]) i += 1 - cdef void _copy_loader(self, int col_from, int col_to): - # Copy the structure, but we may also copy the pointer to the - # context, and we don't want to free() it twice, so mark it as - # "not owned" - self._row_loaders[col_to] = self._row_loaders[col_from] - self._row_loaders[col_to].own_context = 0 - - cdef void _set_loader(self, int col, libpq.Oid oid, int fmt): - pyloader = self.get_load_function(oid, fmt) - - cdef RowLoader *loader = self._row_loaders + col - loader.pyloader = pyloader - self._pyloaders.append(pyloader) - - cdef CLoader cloader = cloaders.get(pyloader) + cdef RowLoader _get_row_loader(self, libpq.Oid oid, int fmt): + cdef RowLoader row_loader = RowLoader() + loader = self.get_loader(oid, fmt) + row_loader.pyloader = loader.load - if cloader is not None: - # The cloader is a normal Python function - loader.cloader = cloader.cloader - return + if isinstance(loader, PyxLoader): + row_loader.pyxloader = loader + else: + row_loader.pyxloader = None - cloader = cloaders.get(getattr(pyloader, '__func__', None)) - if cloader is not None and cloader.get_context is not NULL: - # The cloader is the load() method of a Loader class - # Extract the context from the Loader instance - loader.cloader = cloader.cloader - loader.context = cloader.get_context(pyloader.__self__) - loader.own_context = 1 + return row_loader def get_dumper(self, obj: Any, format: Format) -> "Dumper": key = (type(obj), format) @@ -248,7 +211,7 @@ cdef class Transformer: cdef libpq.PGresult *res = self._pgresult.pgresult_ptr - cdef RowLoader *loader + cdef RowLoader loader cdef int col cdef int length cdef const char *val @@ -262,12 +225,12 @@ cdef class Transformer: continue val = libpq.PQgetvalue(res, crow, col) - loader = self._row_loaders + col - if loader.cloader is not NULL: - pyval = loader.cloader(val, length, loader.context) + loader = self._row_loaders[col] + if loader.pyxloader is not None: + pyval = loader.pyxloader.cload(val, length) else: # TODO: no copy - pyval = (loader.pyloader)(val[:length]) + pyval = loader.pyloader(val[:length]) Py_INCREF(pyval) PyTuple_SET_ITEM(rv, col, pyval) @@ -279,14 +242,14 @@ cdef class Transformer: ) -> Tuple[Any, ...]: cdef list rv = [] cdef int i - cdef RowLoader *loader + cdef RowLoader loader for i in range(len(record)): item = record[i] if item is None: rv.append(None) else: - loader = self._row_loaders + i - rv.append((loader.pyloader)(item)) + loader = self._row_loaders[i] + rv.append(loader.pyloader(item)) return tuple(rv) @@ -308,6 +271,17 @@ cdef class Transformer: func = self._load_funcs[key] = loader(oid, self).load return func + def get_loader(self, oid: int, format: Format) -> "Loader": + key = (oid, format) + try: + return self._loaders_cache[key] + except KeyError: + pass + + loader_cls = self.lookup_loader(*key) + self._loaders_cache[key] = loader = loader_cls(key[0], self) + return loader + def lookup_loader(self, oid: int, format: Format) -> "LoaderType": key = (oid, format)