From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Mon, 21 Dec 2020 02:58:24 +0000 (+0100)
Subject: Release the gil around functions known to be slow
X-Git-Tag: 3.0.dev0~256
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4314414c17bc820190debbd960ae6ef98a1b7e25;p=thirdparty%2Fpsycopg.git

Release the gil around functions known to be slow

Profiling some time ago showed that these functions take relevant time.
Other paths will be reviewed.

https://www.varrazzo.com/blog/2020/05/19/a-trip-into-optimisation/
---

diff --git a/psycopg3_c/psycopg3_c/generators.pyx b/psycopg3_c/psycopg3_c/generators.pyx
index ab840c665..16eb8267b 100644
--- a/psycopg3_c/psycopg3_c/generators.pyx
+++ b/psycopg3_c/psycopg3_c/generators.pyx
@@ -70,6 +70,8 @@ def execute(PGconn pgconn) -> PQGen[List[pq.proto.PGresult]]:
     cdef libpq.PGconn *pgconn_ptr = pgconn.pgconn_ptr
     cdef int status
     cdef libpq.PGnotify *notify
+    cdef libpq.PGresult *pgres
+    cdef int cires, ibres
 
     # Start the generator by sending the connection fd, which won't change
     # during the query process.
@@ -82,19 +84,26 @@ def execute(PGconn pgconn) -> PQGen[List[pq.proto.PGresult]]:
 
         status = yield WAIT_RW
         if status & READY_R:
-            # This call may read notifies which will be saved in the
-            # PGconn buffer and passed to Python later.
-            if 1 != libpq.PQconsumeInput(pgconn_ptr):
+            with nogil:
+                # This call may read notifies which will be saved in the
+                # PGconn buffer and passed to Python later.
+                cires = libpq.PQconsumeInput(pgconn_ptr)
+            if 1 != cires:
                 raise pq.PQerror(
                     f"consuming input failed: {pq.error_message(pgconn)}")
         continue
 
     # Fetching the result
     while 1:
-        if 1 != libpq.PQconsumeInput(pgconn_ptr):
+        with nogil:
+            cires = libpq.PQconsumeInput(pgconn_ptr)
+            if cires == 1:
+                ibres = libpq.PQisBusy(pgconn_ptr)
+
+        if 1 != cires:
             raise pq.PQerror(
                 f"consuming input failed: {pq.error_message(pgconn)}")
-        if libpq.PQisBusy(pgconn_ptr):
+        if ibres:
             yield WAIT_R
             continue
 
@@ -112,12 +121,12 @@ def execute(PGconn pgconn) -> PQGen[List[pq.proto.PGresult]]:
                     break
                 libpq.PQfreemem(notify)
 
-        res = libpq.PQgetResult(pgconn_ptr)
-        if res is NULL:
+        pgres = libpq.PQgetResult(pgconn_ptr)
+        if pgres is NULL:
             break
-        results.append(PGresult._from_ptr(res))
+        results.append(PGresult._from_ptr(pgres))
 
-        status = libpq.PQresultStatus(res)
+        status = libpq.PQresultStatus(pgres)
         if status in (libpq.PGRES_COPY_IN, libpq.PGRES_COPY_OUT, libpq.PGRES_COPY_BOTH):
             # After entering copy mode the libpq will create a phony result
             # for every request so let's break the endless loop.
diff --git a/psycopg3_c/psycopg3_c/libpq.pxd b/psycopg3_c/psycopg3_c/libpq.pxd
index a9e241d59..d4498c4c6 100644
--- a/psycopg3_c/psycopg3_c/libpq.pxd
+++ b/psycopg3_c/psycopg3_c/libpq.pxd
@@ -224,8 +224,8 @@ cdef extern from "libpq-fe.h":
     int PQsendDescribePrepared(PGconn *conn, const char *stmtName)
     int PQsendDescribePortal(PGconn *conn, const char *portalName)
     PGresult *PQgetResult(PGconn *conn)
-    int PQconsumeInput(PGconn *conn)
-    int PQisBusy(PGconn *conn)
+    int PQconsumeInput(PGconn *conn) nogil
+    int PQisBusy(PGconn *conn) nogil
     int PQsetnonblocking(PGconn *conn, int arg)
     int PQisnonblocking(const PGconn *conn)
     int PQflush(PGconn *conn)
@@ -236,7 +236,7 @@ cdef extern from "libpq-fe.h":
     int PQcancel(PGcancel *cancel, char *errbuf, int errbufsize)
 
     # 33.8. Asynchronous Notification
-    PGnotify *PQnotifies(PGconn *conn)
+    PGnotify *PQnotifies(PGconn *conn) nogil
 
     # 33.9. Functions Associated with the COPY Command
     int PQputCopyData(PGconn *conn, const char *buffer, int nbytes)
@@ -244,7 +244,7 @@ cdef extern from "libpq-fe.h":
     int PQgetCopyData(PGconn *conn, char **buffer, int async)
 
     # 33.11. Miscellaneous Functions
-    void PQfreemem(void *ptr)
+    void PQfreemem(void *ptr) nogil
     void PQconninfoFree(PQconninfoOption *connOptions)
     PGresult *PQmakeEmptyPGresult(PGconn *conn, ExecStatusType status)
     int PQsetResultAttrs(PGresult *res, int numAttributes, PGresAttDesc *attDescs)
diff --git a/psycopg3_c/psycopg3_c/pq_cython.pyx b/psycopg3_c/psycopg3_c/pq_cython.pyx
index fac39a5b2..6dd973ac1 100644
--- a/psycopg3_c/psycopg3_c/pq_cython.pyx
+++ b/psycopg3_c/psycopg3_c/pq_cython.pyx
@@ -41,7 +41,7 @@ def version():
     return impl.PQlibVersion()
 
 
-cdef void notice_receiver(void *arg, const impl.PGresult *res_ptr):
+cdef void notice_receiver(void *arg, const impl.PGresult *res_ptr) with gil:
     cdef PGconn pgconn = <object>arg
     if pgconn.notice_handler is None:
         return
@@ -401,7 +401,10 @@ cdef class PGconn:
             raise PQerror(f"consuming input failed: {error_message(self)}")
 
     def is_busy(self) -> int:
-        return impl.PQisBusy(self.pgconn_ptr)
+        cdef int rv
+        with nogil:
+            rv = impl.PQisBusy(self.pgconn_ptr)
+        return rv
 
     @property
     def nonblocking(self) -> int:
@@ -425,7 +428,10 @@ cdef class PGconn:
         return PGcancel._from_ptr(ptr)
 
     def notifies(self) -> Optional[PGnotify]:
-        cdef impl.PGnotify *ptr = impl.PQnotifies(self.pgconn_ptr)
+        cdef impl.PGnotify *ptr
+        with nogil:
+            ptr = impl.PQnotifies(self.pgconn_ptr)
+
         if ptr:
             ret = PGnotify(ptr.relname, ptr.be_pid, ptr.extra)
             impl.PQfreemem(ptr)