From f1be7ecde39070f0e8de1aa031461c053e1dae33 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 13 Sep 2025 15:59:56 +0200 Subject: [PATCH] fix: basic Cython improvements to row iteration Add a module level control, rename var, set some vars types --- psycopg_c/psycopg_c/_psycopg.pyx | 1 + psycopg_c/psycopg_c/_psycopg/transform.pyx | 22 ++++++++++++++-------- tests/test_row_pagination_c.py | 19 ++++++++----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/psycopg_c/psycopg_c/_psycopg.pyx b/psycopg_c/psycopg_c/_psycopg.pyx index 731ba581c..631909e0c 100644 --- a/psycopg_c/psycopg_c/_psycopg.pyx +++ b/psycopg_c/psycopg_c/_psycopg.pyx @@ -25,6 +25,7 @@ PG_AUTO = _py_Format.AUTO PG_TEXT = _py_Format.TEXT PG_BINARY = _py_Format.BINARY +_load_rows_page_size = 100 cdef extern from *: """ diff --git a/psycopg_c/psycopg_c/_psycopg/transform.pyx b/psycopg_c/psycopg_c/_psycopg/transform.pyx index 0be56b7f2..08de3dcdd 100644 --- a/psycopg_c/psycopg_c/_psycopg/transform.pyx +++ b/psycopg_c/psycopg_c/_psycopg/transform.pyx @@ -97,7 +97,7 @@ cdef class Transformer: cdef dict _oid_types - cdef public int _page_size + cdef public int _load_rows_page_size def __cinit__(self, context: "AdaptContext" | None = None): if context is not None: @@ -110,7 +110,7 @@ cdef class Transformer: self.types = self.formats = None self._none_oid = -1 - self._page_size = 100 + self._load_rows_page_size = _load_rows_page_size @classmethod def from_context(cls, context: "AdaptContext" | None): @@ -434,12 +434,14 @@ cdef class Transformer: f"rows must be included between 0 and {self._ntuples}" ) + if self._load_rows_page_size <= 0: + raise ValueError(f"_load_rows_page_size must be positive") + cdef libpq.PGresult *res = self._pgresult._pgresult_ptr # cheeky access to the internal PGresult structure cdef pg_result_int *ires = res - cdef int row - cdef int col + cdef int row, col, pr0, pr1 cdef PGresAttValue *attval cdef object record # not 'tuple' as it would check on assignment @@ -447,15 +449,17 @@ cdef class Transformer: cdef PyObject *loader # borrowed RowLoader cdef PyObject *brecord # borrowed + row_loaders = self._row_loaders # avoid an incref/decref per item - cdef int page_size = self._page_size - for pr0 in range(row0, row1, page_size): - pr1 = min(pr0 + page_size, row1) + pr0 = row0 + while pr0 < row1: + pr1 = min(pr0 + self._load_rows_page_size, row1) for row in range(pr0, pr1): record = PyTuple_New(self._nfields) Py_INCREF(record) - PyList_SET_ITEM(records, row, record) + PyList_SET_ITEM(records, row - row0, record) + for col in range(self._nfields): loader = PyList_GET_ITEM(row_loaders, col) if (loader).cloader is not None: @@ -488,6 +492,8 @@ cdef class Transformer: Py_INCREF(pyval) PyTuple_SET_ITEM(brecord, col, pyval) + pr0 += self._load_rows_page_size + if make_row is not tuple: for i in range(row1 - row0): brecord = PyList_GET_ITEM(records, i) diff --git a/tests/test_row_pagination_c.py b/tests/test_row_pagination_c.py index 6b73831ec..c922d18bc 100644 --- a/tests/test_row_pagination_c.py +++ b/tests/test_row_pagination_c.py @@ -1,16 +1,13 @@ import pytest -@pytest.mark.parametrize('pagesize', [1, 2, 3, 5, 7]) +import psycopg._cmodule + + +@pytest.mark.parametrize("pagesize", [1, 2, 3, 5, 7]) +@pytest.mark.skipif(not psycopg._cmodule._psycopg, reason="C module test only") def test_pagesize_c(conn, pagesize): - from psycopg._cmodule import _psycopg - if not _psycopg: - return - cur = conn.cursor() - cur.execute("SELECT *, 'abc' FROM generate_series(1, 10)") - cur._tx._page_size = pagesize + cur = conn.execute("SELECT *, 'abc' FROM generate_series(1, 10)") + cur._tx._load_rows_page_size = pagesize result = cur.fetchall() - expected = [ - (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'abc'), (5, 'abc'), - (6, 'abc'), (7, 'abc'), (8, 'abc'), (9, 'abc'), (10, 'abc') - ] + expected = [(i, "abc") for i in range(1, 11)] assert result == expected -- 2.47.3