From ca46caede4b8d846f3cd48e642922ae821d0be2b Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sun, 4 Dec 2022 23:25:14 -0500 Subject: [PATCH] adjustments for unreliable gc sporadic (and at the moment persistent) test failures related to aiosqlite seem to have in common that Python gc stops working fully when we run a lot of tests with aiosqlite. The failures are not limited to aiosqlite as they are more involving places where we assume or expect gc.collect() to get rid of things, and it doesn't. Identify (based on reproducible case on the d3 CI runner) the spots where this happens and add fixes. test/orm/test_transaction.py test_gced_delete_on_rollback has always been a very sensitive test with a lot of issues, so here we move it to the test_memusage suite and limit it only to when the memusage suite is running. Change-Id: I683412d0effe8732c45980b40722e5bb63431177 --- doc/build/orm/queryguide/_deferred_setup.rst | 1 + doc/build/orm/queryguide/select.rst | 4 ++ test/aaa_profiling/test_memusage.py | 54 ++++++++++++++++++++ test/orm/test_transaction.py | 39 -------------- test/requirements.py | 1 + 5 files changed, 60 insertions(+), 39 deletions(-) diff --git a/doc/build/orm/queryguide/_deferred_setup.rst b/doc/build/orm/queryguide/_deferred_setup.rst index e75630c46e..2675c93411 100644 --- a/doc/build/orm/queryguide/_deferred_setup.rst +++ b/doc/build/orm/queryguide/_deferred_setup.rst @@ -100,5 +100,6 @@ This page illustrates the mappings and fixture data used by the ... ) >>> session.commit() BEGIN ... COMMIT + >>> session.close() >>> conn.begin() BEGIN ... diff --git a/doc/build/orm/queryguide/select.rst b/doc/build/orm/queryguide/select.rst index 7967bb4d57..55c3ae94ea 100644 --- a/doc/build/orm/queryguide/select.rst +++ b/doc/build/orm/queryguide/select.rst @@ -1001,6 +1001,10 @@ which belonged to "sandy": Relationship Instance Comparison Operators ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. comment + + >>> session.expunge_all() + The :func:`_orm.relationship`-bound attribute also offers a few SQL construction implementations that are geared towards filtering a :func:`_orm.relationship`-bound attribute in terms of a specific instance of a related object, which can unpack diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py index 1162a54afd..4756a3fd5c 100644 --- a/test/aaa_profiling/test_memusage.py +++ b/test/aaa_profiling/test_memusage.py @@ -18,6 +18,7 @@ from sqlalchemy import util from sqlalchemy.engine import result from sqlalchemy.engine.processors import to_decimal_processor_factory from sqlalchemy.orm import aliased +from sqlalchemy.orm import attributes from sqlalchemy.orm import clear_mappers from sqlalchemy.orm import configure_mappers from sqlalchemy.orm import declarative_base @@ -1724,3 +1725,56 @@ class CycleTest(_fixtures.FixtureTest): s.close() go() + + +@testing.add_to_marker.memory_intensive +class MiscMemoryIntensiveTests(fixtures.TestBase): + @testing.fixture + def user_fixture(self, decl_base): + class User(decl_base): + __tablename__ = "user" + + id = Column(Integer, primary_key=True) + name = Column(String(50)) + + decl_base.metadata.create_all(testing.db) + yield User + + @testing.requires.predictable_gc + def test_gced_delete_on_rollback(self, user_fixture): + User = user_fixture + + s = fixture_session() + u1 = User(name="ed") + s.add(u1) + s.commit() + + s.delete(u1) + u1_state = attributes.instance_state(u1) + assert u1_state in s.identity_map.all_states() + assert u1_state in s._deleted + s.flush() + assert u1_state not in s.identity_map.all_states() + assert u1_state not in s._deleted + del u1 + gc_collect() + gc_collect() + gc_collect() + assert u1_state.obj() is None + + s.rollback() + # new in 1.1, not in identity map if the object was + # gc'ed and we restore snapshot; we've changed update_impl + # to just skip this object + assert u1_state not in s.identity_map.all_states() + + # in any version, the state is replaced by the query + # because the identity map would switch it + u1 = s.query(User).filter_by(name="ed").one() + assert u1_state not in s.identity_map.all_states() + + eq_(s.scalar(select(func.count("*")).select_from(User.__table__)), 1) + s.delete(u1) + s.flush() + eq_(s.scalar(select(func.count("*")).select_from(User.__table__)), 0) + s.commit() diff --git a/test/orm/test_transaction.py b/test/orm/test_transaction.py index 2f08080ada..f66908fc9c 100644 --- a/test/orm/test_transaction.py +++ b/test/orm/test_transaction.py @@ -1249,45 +1249,6 @@ class AutoExpireTest(_LocalFixture): assert u1 in s assert u1 not in s.deleted - @testing.requires.predictable_gc - def test_gced_delete_on_rollback(self): - User, users = self.classes.User, self.tables.users - - s = fixture_session() - u1 = User(name="ed") - s.add(u1) - s.commit() - - s.delete(u1) - u1_state = attributes.instance_state(u1) - assert u1_state in s.identity_map.all_states() - assert u1_state in s._deleted - s.flush() - assert u1_state not in s.identity_map.all_states() - assert u1_state not in s._deleted - del u1 - gc_collect() - gc_collect() - gc_collect() - assert u1_state.obj() is None - - s.rollback() - # new in 1.1, not in identity map if the object was - # gc'ed and we restore snapshot; we've changed update_impl - # to just skip this object - assert u1_state not in s.identity_map.all_states() - - # in any version, the state is replaced by the query - # because the identity map would switch it - u1 = s.query(User).filter_by(name="ed").one() - assert u1_state not in s.identity_map.all_states() - - eq_(s.scalar(select(func.count("*")).select_from(users)), 1) - s.delete(u1) - s.flush() - eq_(s.scalar(select(func.count("*")).select_from(users)), 0) - s.commit() - def test_trans_deleted_cleared_on_rollback(self): User = self.classes.User s = fixture_session() diff --git a/test/requirements.py b/test/requirements.py index e56c944ffc..5276593c93 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -397,6 +397,7 @@ class DefaultRequirements(SuiteRequirements): [ no_support("oracle", "Oracle XE usually can't handle these"), no_support("mssql+pyodbc", "MS ODBC drivers struggle"), + no_support("+aiosqlite", "very unreliable driver"), self._running_on_windows(), ] ) -- 2.47.2