fix: basic Cython improvements to row iteration

Add a module level control, rename var, set some vars types

diff --git a/psycopg_c/psycopg_c/_psycopg.pyx b/psycopg_c/psycopg_c/_psycopg.pyx
index 731ba581c4c60fde811e243d623feaf13978ca31..631909e0cc6570c31d6026c2e1b31c4c1049b3e1 100644 (file)
--- a/psycopg_c/psycopg_c/_psycopg.pyx
+++ b/psycopg_c/psycopg_c/_psycopg.pyx
@@ -25,6 +25,7 @@ PG_AUTO = _py_Format.AUTO
 PG_TEXT = _py_Format.TEXT
 PG_BINARY = _py_Format.BINARY
 
+_load_rows_page_size = 100
 
 cdef extern from *:
     """

diff --git a/psycopg_c/psycopg_c/_psycopg/transform.pyx b/psycopg_c/psycopg_c/_psycopg/transform.pyx
index 0be56b7f243177ad4ac9006ba5832587416095d2..08de3dcdd4861510bc04bf418d5f7293eab5c46d 100644 (file)
--- a/psycopg_c/psycopg_c/_psycopg/transform.pyx
+++ b/psycopg_c/psycopg_c/_psycopg/transform.pyx
@@ -97,7 +97,7 @@ cdef class Transformer:
 
     cdef dict _oid_types
 
-    cdef public int _page_size
+    cdef public int _load_rows_page_size
 
     def __cinit__(self, context: "AdaptContext" | None = None):
         if context is not None:
@@ -110,7 +110,7 @@ cdef class Transformer:
 
         self.types = self.formats = None
         self._none_oid = -1
-        self._page_size = 100
+        self._load_rows_page_size = _load_rows_page_size
 
     @classmethod
     def from_context(cls, context: "AdaptContext" | None):
@@ -434,12 +434,14 @@ cdef class Transformer:
                 f"rows must be included between 0 and {self._ntuples}"
             )
 
+        if self._load_rows_page_size <= 0:
+            raise ValueError(f"_load_rows_page_size  must be positive")
+
         cdef libpq.PGresult *res = self._pgresult._pgresult_ptr
         # cheeky access to the internal PGresult structure
         cdef pg_result_int *ires = <pg_result_int*>res
 
-        cdef int row
-        cdef int col
+        cdef int row, col, pr0, pr1
         cdef PGresAttValue *attval
         cdef object record  # not 'tuple' as it would check on assignment
 
@@ -447,15 +449,17 @@ cdef class Transformer:
 
         cdef PyObject *loader  # borrowed RowLoader
         cdef PyObject *brecord  # borrowed
+
         row_loaders = self._row_loaders  # avoid an incref/decref per item
 
-        cdef int page_size = self._page_size
-        for pr0 in range(row0, row1, page_size):
-            pr1 = min(pr0 + page_size, row1)
+        pr0 = row0
+        while pr0 < row1:
+            pr1 = min(pr0 + self._load_rows_page_size, row1)
             for row in range(pr0, pr1):
                 record = PyTuple_New(self._nfields)
                 Py_INCREF(record)
-                PyList_SET_ITEM(records, row, record)
+                PyList_SET_ITEM(records, row - row0, record)
+
             for col in range(self._nfields):
                 loader = PyList_GET_ITEM(row_loaders, col)
                 if (<RowLoader>loader).cloader is not None:
@@ -488,6 +492,8 @@ cdef class Transformer:
                         Py_INCREF(pyval)
                         PyTuple_SET_ITEM(<object>brecord, col, pyval)
 
+            pr0 += self._load_rows_page_size
+
         if make_row is not tuple:
             for i in range(row1 - row0):
                 brecord = PyList_GET_ITEM(records, i)

diff --git a/tests/test_row_pagination_c.py b/tests/test_row_pagination_c.py
index 6b73831ecc10debeb80114e74a7ecb4eb7d8abee..c922d18bcb834d68e969b2a7d792a36fffc1a2c7 100644 (file)
--- a/tests/test_row_pagination_c.py
+++ b/tests/test_row_pagination_c.py
@@ -1,16 +1,13 @@
 import pytest
 
-@pytest.mark.parametrize('pagesize', [1, 2, 3, 5, 7])
+import psycopg._cmodule
+
+
+@pytest.mark.parametrize("pagesize", [1, 2, 3, 5, 7])
+@pytest.mark.skipif(not psycopg._cmodule._psycopg, reason="C module test only")
 def test_pagesize_c(conn, pagesize):
-    from psycopg._cmodule import _psycopg
-    if not _psycopg:
-        return
-    cur = conn.cursor()
-    cur.execute("SELECT *, 'abc' FROM generate_series(1, 10)")
-    cur._tx._page_size = pagesize
+    cur = conn.execute("SELECT *, 'abc' FROM generate_series(1, 10)")
+    cur._tx._load_rows_page_size = pagesize
     result = cur.fetchall()
-    expected = [
-        (1, 'abc'), (2, 'abc'), (3, 'abc'), (4, 'abc'), (5, 'abc'),
-        (6, 'abc'), (7, 'abc'), (8, 'abc'), (9, 'abc'), (10, 'abc')
-    ]
+    expected = [(i, "abc") for i in range(1, 11)]
     assert result == expected