Added max_row_buffer attribute to the context execution options and use

it to prevent excess memory usage with yield_per

diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py
index 56c81c93e99bf4617a648c49a00b1d4fb5fdf39b..41b30c98311b2090a1967b8284f57119db44c1b4 100644 (file)
--- a/lib/sqlalchemy/engine/result.py
+++ b/lib/sqlalchemy/engine/result.py
@@ -1067,7 +1067,7 @@ class BufferedRowResultProxy(ResultProxy):
 
     The pre-fetching behavior fetches only one row initially, and then
     grows its buffer size by a fixed amount with each successive need
-    for additional rows up to a size of 100.
+    for additional rows up to a size of 1000.
     """
 
     def _init_metadata(self):
@@ -1083,7 +1083,10 @@ class BufferedRowResultProxy(ResultProxy):
         5: 10,
         10: 20,
         20: 50,
-        50: 100
+        50: 100,
+        100: 250,
+        250: 500,
+        500: 1000
     }
 
     def __buffer_rows(self):
@@ -1092,6 +1095,8 @@ class BufferedRowResultProxy(ResultProxy):
         size = getattr(self, '_bufsize', 1)
         self.__rowbuffer = collections.deque(self.cursor.fetchmany(size))
         self._bufsize = self.size_growth.get(size, size)
+        if self.context.execution_options.get('max_row_buffer') is not None:
+            self._bufsize = min(self.context.execution_options['max_row_buffer'], self._bufsize)
 
     def _soft_close(self, **kw):
         self.__rowbuffer.clear()

diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py
index 4f8c86a144a572f8196a12e067fe9e3d71d59ce3..8b3df08e762ad58d6d634f84d0378558568a93ab 100644 (file)
--- a/lib/sqlalchemy/orm/query.py
+++ b/lib/sqlalchemy/orm/query.py
@@ -756,7 +756,8 @@ class Query(object):
         """
         self._yield_per = count
         self._execution_options = self._execution_options.union(
-            {"stream_results": True})
+            {"stream_results": True,
+             "max_row_buffer": count})
 
     def get(self, ident):
         """Return an instance based on the given primary key identifier,