improve fetchmany performance when using deque

Change-Id: Id30e770eb44eafd3e939c4076b639e8e6962c54b
(cherry picked from commit 319304e7c9e5c6c2e42513b81f85aa6b238495b5)

diff --git a/lib/sqlalchemy/connectors/asyncio.py b/lib/sqlalchemy/connectors/asyncio.py
index 0b44f23a025b247533733c5004218eb1fa8ba142..8dc198cf8e9491c94469c31b9ac11df1c3c42032 100644 (file)
--- a/lib/sqlalchemy/connectors/asyncio.py
+++ b/lib/sqlalchemy/connectors/asyncio.py
@@ -11,7 +11,6 @@
 from __future__ import annotations
 
 import collections
-import itertools
 
 from ..engine import AdaptedConnection
 from ..util.concurrency import asyncio
@@ -114,11 +113,8 @@ class AsyncAdapt_dbapi_cursor:
     def fetchmany(self, size=None):
         if size is None:
             size = self.arraysize
-
-        rr = iter(self._rows)
-        retval = list(itertools.islice(rr, 0, size))
-        self._rows = collections.deque(rr)
-        return retval
+        rr = self._rows
+        return [rr.popleft() for _ in range(min(size, len(rr)))]
 
     def fetchall(self):
         retval = list(self._rows)

diff --git a/lib/sqlalchemy/engine/cursor.py b/lib/sqlalchemy/engine/cursor.py
index 71767db74edcf07f48d2417547489f9d0dcd735b..a885aca8e3b56ff59c2c684534dea24f5160ee85 100644 (file)
--- a/lib/sqlalchemy/engine/cursor.py
+++ b/lib/sqlalchemy/engine/cursor.py
@@ -1246,8 +1246,9 @@ class BufferedRowCursorFetchStrategy(CursorFetchStrategy):
         if size is None:
             return self.fetchall(result, dbapi_cursor)
 
-        buf = list(self._rowbuffer)
-        lb = len(buf)
+        rb = self._rowbuffer
+        lb = len(rb)
+        close = False
         if size > lb:
             try:
                 new = dbapi_cursor.fetchmany(size - lb)
@@ -1255,13 +1256,15 @@ class BufferedRowCursorFetchStrategy(CursorFetchStrategy):
                 self.handle_exception(result, dbapi_cursor, e)
             else:
                 if not new:
-                    result._soft_close()
+                    # defer closing since it may clear the row buffer
+                    close = True
                 else:
-                    buf.extend(new)
+                    rb.extend(new)
 
-        result = buf[0:size]
-        self._rowbuffer = collections.deque(buf[size:])
-        return result
+        res = [rb.popleft() for _ in range(min(size, len(rb)))]
+        if close:
+            result._soft_close()
+        return res
 
     def fetchall(self, result, dbapi_cursor):
         try:
@@ -1315,9 +1318,8 @@ class FullyBufferedCursorFetchStrategy(CursorFetchStrategy):
         if size is None:
             return self.fetchall(result, dbapi_cursor)
 
-        buf = list(self._rowbuffer)
-        rows = buf[0:size]
-        self._rowbuffer = collections.deque(buf[size:])
+        rb = self._rowbuffer
+        rows = [rb.popleft() for _ in range(min(size, len(rb)))]
         if not rows:
             result._soft_close()
         return rows