adjustments for unreliable gc

sporadic (and at the moment persistent) test failures
related to aiosqlite seem to have in common that Python
gc stops working fully when we run a lot of tests with
aiosqlite.  The failures are not limited to aiosqlite
as they are more involving places where we assume or
expect gc.collect() to get rid of things, and it doesn't.

Identify (based on reproducible case on the d3 CI runner)
the spots where this happens and add fixes.

test/orm/test_transaction.py test_gced_delete_on_rollback
has always been a very sensitive test with a lot of issues,
so here we move it to the test_memusage suite and limit
it only to when the memusage suite is running.

Change-Id: I683412d0effe8732c45980b40722e5bb63431177

diff --git a/doc/build/orm/queryguide/_deferred_setup.rst b/doc/build/orm/queryguide/_deferred_setup.rst
index e75630c46ea134e8a3ef6aabf91e4bb49715cb64..2675c93411673175d12e3544079eb935ac79af22 100644 (file)
--- a/doc/build/orm/queryguide/_deferred_setup.rst
+++ b/doc/build/orm/queryguide/_deferred_setup.rst
@@ -100,5 +100,6 @@ This page illustrates the mappings and fixture data used by the
     ... )
     >>> session.commit()
     BEGIN ... COMMIT
+    >>> session.close()
     >>> conn.begin()
     BEGIN ...

diff --git a/doc/build/orm/queryguide/select.rst b/doc/build/orm/queryguide/select.rst
index 7967bb4d57c2964bdaf1880143de50af9b187042..55c3ae94ea711fac4e229d744abe70c349182126 100644 (file)
--- a/doc/build/orm/queryguide/select.rst
+++ b/doc/build/orm/queryguide/select.rst
@@ -1001,6 +1001,10 @@ which belonged to "sandy":
 Relationship Instance Comparison Operators
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+.. comment
+
+    >>> session.expunge_all()
+
 The :func:`_orm.relationship`-bound attribute also offers a few SQL construction
 implementations that are geared towards filtering a :func:`_orm.relationship`-bound
 attribute in terms of a specific instance of a related object, which can unpack

diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py
index 1162a54afda7a319621076529ccb9b16d192ce88..4756a3fd5c3634ed81483c134afc8fd31394a277 100644 (file)
--- a/test/aaa_profiling/test_memusage.py
+++ b/test/aaa_profiling/test_memusage.py
@@ -18,6 +18,7 @@ from sqlalchemy import util
 from sqlalchemy.engine import result
 from sqlalchemy.engine.processors import to_decimal_processor_factory
 from sqlalchemy.orm import aliased
+from sqlalchemy.orm import attributes
 from sqlalchemy.orm import clear_mappers
 from sqlalchemy.orm import configure_mappers
 from sqlalchemy.orm import declarative_base
@@ -1724,3 +1725,56 @@ class CycleTest(_fixtures.FixtureTest):
             s.close()
 
         go()
+
+
+@testing.add_to_marker.memory_intensive
+class MiscMemoryIntensiveTests(fixtures.TestBase):
+    @testing.fixture
+    def user_fixture(self, decl_base):
+        class User(decl_base):
+            __tablename__ = "user"
+
+            id = Column(Integer, primary_key=True)
+            name = Column(String(50))
+
+        decl_base.metadata.create_all(testing.db)
+        yield User
+
+    @testing.requires.predictable_gc
+    def test_gced_delete_on_rollback(self, user_fixture):
+        User = user_fixture
+
+        s = fixture_session()
+        u1 = User(name="ed")
+        s.add(u1)
+        s.commit()
+
+        s.delete(u1)
+        u1_state = attributes.instance_state(u1)
+        assert u1_state in s.identity_map.all_states()
+        assert u1_state in s._deleted
+        s.flush()
+        assert u1_state not in s.identity_map.all_states()
+        assert u1_state not in s._deleted
+        del u1
+        gc_collect()
+        gc_collect()
+        gc_collect()
+        assert u1_state.obj() is None
+
+        s.rollback()
+        # new in 1.1, not in identity map if the object was
+        # gc'ed and we restore snapshot; we've changed update_impl
+        # to just skip this object
+        assert u1_state not in s.identity_map.all_states()
+
+        # in any version, the state is replaced by the query
+        # because the identity map would switch it
+        u1 = s.query(User).filter_by(name="ed").one()
+        assert u1_state not in s.identity_map.all_states()
+
+        eq_(s.scalar(select(func.count("*")).select_from(User.__table__)), 1)
+        s.delete(u1)
+        s.flush()
+        eq_(s.scalar(select(func.count("*")).select_from(User.__table__)), 0)
+        s.commit()

diff --git a/test/orm/test_transaction.py b/test/orm/test_transaction.py
index 2f08080adab0eb7956641e27794c5312edca789d..f66908fc9c9d73bdb1954148a38c25dfc2cc72d2 100644 (file)
--- a/test/orm/test_transaction.py
+++ b/test/orm/test_transaction.py
@@ -1249,45 +1249,6 @@ class AutoExpireTest(_LocalFixture):
         assert u1 in s
         assert u1 not in s.deleted
 
-    @testing.requires.predictable_gc
-    def test_gced_delete_on_rollback(self):
-        User, users = self.classes.User, self.tables.users
-
-        s = fixture_session()
-        u1 = User(name="ed")
-        s.add(u1)
-        s.commit()
-
-        s.delete(u1)
-        u1_state = attributes.instance_state(u1)
-        assert u1_state in s.identity_map.all_states()
-        assert u1_state in s._deleted
-        s.flush()
-        assert u1_state not in s.identity_map.all_states()
-        assert u1_state not in s._deleted
-        del u1
-        gc_collect()
-        gc_collect()
-        gc_collect()
-        assert u1_state.obj() is None
-
-        s.rollback()
-        # new in 1.1, not in identity map if the object was
-        # gc'ed and we restore snapshot; we've changed update_impl
-        # to just skip this object
-        assert u1_state not in s.identity_map.all_states()
-
-        # in any version, the state is replaced by the query
-        # because the identity map would switch it
-        u1 = s.query(User).filter_by(name="ed").one()
-        assert u1_state not in s.identity_map.all_states()
-
-        eq_(s.scalar(select(func.count("*")).select_from(users)), 1)
-        s.delete(u1)
-        s.flush()
-        eq_(s.scalar(select(func.count("*")).select_from(users)), 0)
-        s.commit()
-
     def test_trans_deleted_cleared_on_rollback(self):
         User = self.classes.User
         s = fixture_session()

diff --git a/test/requirements.py b/test/requirements.py
index e56c944ffcce31fd546c97ce29ad90b76df3cd26..5276593c93b712d0bb191edc686762c0dab60acf 100644 (file)
--- a/test/requirements.py
+++ b/test/requirements.py
@@ -397,6 +397,7 @@ class DefaultRequirements(SuiteRequirements):
             [
                 no_support("oracle", "Oracle XE usually can't handle these"),
                 no_support("mssql+pyodbc", "MS ODBC drivers struggle"),
+                no_support("+aiosqlite", "very unreliable driver"),
                 self._running_on_windows(),
             ]
         )