From 7e87605140ee5cdd602f86dfd6931cddda0fde0e Mon Sep 17 00:00:00 2001 From: Jason Kirtland Date: Wed, 1 Apr 2009 01:52:16 +0000 Subject: [PATCH] Migrated gc.collect hack to testlib.compat --- test/engine/pool.py | 5 +++-- test/engine/reconnect.py | 12 +++++++----- test/ext/associationproxy.py | 5 +++-- test/orm/attributes.py | 3 ++- test/orm/expire.py | 5 +++-- test/orm/session.py | 15 ++++++++------- test/orm/transaction.py | 4 +++- test/perf/masscreate.py | 2 -- test/perf/masscreate2.py | 4 ++-- test/perf/massload.py | 2 +- test/perf/masssave.py | 1 + test/perf/objselectspeed.py | 12 ++++++------ test/perf/objupdatespeed.py | 6 ++++-- test/perf/sessions.py | 10 +++++----- test/profiling/memusage.py | 12 +++++++----- test/testlib/__init__.py | 3 +-- test/testlib/compat.py | 16 +++++++++++++++- test/testlib/profiling.py | 6 +++--- test/testlib/sa_gc.py | 14 -------------- 19 files changed, 74 insertions(+), 63 deletions(-) delete mode 100644 test/testlib/sa_gc.py diff --git a/test/engine/pool.py b/test/engine/pool.py index 33ef80ff8f..93c6fe8dd0 100644 --- a/test/engine/pool.py +++ b/test/engine/pool.py @@ -2,7 +2,8 @@ import testenv; testenv.configure_for_tests() import threading, time from sqlalchemy import pool, interfaces, create_engine import testlib.sa as tsa -from testlib import TestBase, gc, testing +from testlib import TestBase, testing +from testlib.compat import gc_collect mcid = 1 @@ -542,7 +543,7 @@ class QueuePoolTest(PoolTestBase): c2 = None del c1 del c2 - gc.collect() + gc_collect() assert p.checkedout() == 0 c3 = p.connect() assert c3 is not None diff --git a/test/engine/reconnect.py b/test/engine/reconnect.py index e1d5744db5..1203a152b7 100644 --- a/test/engine/reconnect.py +++ b/test/engine/reconnect.py @@ -1,9 +1,11 @@ import testenv; testenv.configure_for_tests() +import time import weakref from testlib.sa import select, MetaData, Table, Column, Integer, String, pool import testlib.sa as tsa -from testlib import TestBase, testing, engines, gc -import time +from testlib import TestBase, testing, engines +from testlib.compat import gc_collect + class MockDisconnect(Exception): pass @@ -95,7 +97,7 @@ class MockReconnectTest(TestBase): assert id(db.pool) != pid # ensure all connections closed (pool was recycled) - gc.collect() + gc_collect() assert len(dbapi.connections) == 0 conn =db.connect() @@ -115,7 +117,7 @@ class MockReconnectTest(TestBase): pass # assert was invalidated - gc.collect() + gc_collect() assert len(dbapi.connections) == 0 assert not conn.closed assert conn.invalidated @@ -165,7 +167,7 @@ class MockReconnectTest(TestBase): assert conn.invalidated # ensure all connections closed (pool was recycled) - gc.collect() + gc_collect() assert len(dbapi.connections) == 0 # test reconnects diff --git a/test/ext/associationproxy.py b/test/ext/associationproxy.py index 64683819bc..8b2d9e5548 100644 --- a/test/ext/associationproxy.py +++ b/test/ext/associationproxy.py @@ -4,6 +4,7 @@ from sqlalchemy.orm import * from sqlalchemy.orm.collections import collection from sqlalchemy.ext.associationproxy import * from testlib import * +from testlib.compat import gc_collect class DictCollection(dict): @@ -864,7 +865,7 @@ class ReconstitutionTest(TestBase): add_child('p1', 'c1') - gc.collect() + gc_collect() add_child('p1', 'c2') session.flush() @@ -877,7 +878,7 @@ class ReconstitutionTest(TestBase): p.kids.extend(['c1', 'c2']) p_copy = copy.copy(p) del p - gc.collect() + gc_collect() assert set(p_copy.kids) == set(['c1', 'c2']), p.kids diff --git a/test/orm/attributes.py b/test/orm/attributes.py index be8cbf714c..46d944cbc3 100644 --- a/test/orm/attributes.py +++ b/test/orm/attributes.py @@ -5,6 +5,7 @@ from sqlalchemy.orm.collections import collection from sqlalchemy.orm.interfaces import AttributeExtension from sqlalchemy import exc as sa_exc from testlib import * +from testlib.compat import gc_collect from testlib.testing import eq_ from orm import _base @@ -115,7 +116,7 @@ class AttributesTest(_base.ORMTest): f.bar = "foo" assert state.dict == {'bar':'foo', state.manager.STATE_ATTR:state} del f - gc.collect() + gc_collect() assert state.obj() is None assert state.dict == {} diff --git a/test/orm/expire.py b/test/orm/expire.py index bbc011f875..34e9a57fe3 100644 --- a/test/orm/expire.py +++ b/test/orm/expire.py @@ -1,7 +1,8 @@ """Attribute/instance expiration, deferral of attributes, etc.""" import testenv; testenv.configure_for_tests() -from testlib import gc, sa, testing +from testlib import sa, testing +from testlib.compat import gc_collect from testlib.sa import Table, Column, Integer, String, ForeignKey, exc as sa_exc from testlib.sa.orm import mapper, relation, create_session, attributes, deferred from orm import _base, _fixtures @@ -662,7 +663,7 @@ class ExpireTest(_fixtures.FixtureTest): assert self.static.user_address_result == userlist assert len(list(sess)) == 9 sess.expire_all() - gc.collect() + gc_collect() assert len(list(sess)) == 4 # since addresses were gc'ed userlist = sess.query(User).order_by(User.id).all() diff --git a/test/orm/session.py b/test/orm/session.py index dc5bf25784..5a2229b16c 100644 --- a/test/orm/session.py +++ b/test/orm/session.py @@ -2,7 +2,8 @@ import testenv; testenv.configure_for_tests() import inspect import pickle from sqlalchemy.orm import create_session, sessionmaker, attributes -from testlib import engines, gc, sa, testing, config +from testlib import engines, sa, testing, config +from testlib.compat import gc_collect from testlib.sa import Table, Column, Integer, String, Sequence from testlib.sa.orm import mapper, relation, backref from testlib.testing import eq_ @@ -225,7 +226,7 @@ class SessionTest(_fixtures.FixtureTest): u = sess.query(User).get(u.id) q = sess.query(Address).filter(Address.user==u) del u - gc.collect() + gc_collect() eq_(q.one(), Address(email_address='foo')) @@ -757,18 +758,18 @@ class SessionTest(_fixtures.FixtureTest): user = s.query(User).one() del user - gc.collect() + gc_collect() assert len(s.identity_map) == 0 user = s.query(User).one() user.name = 'fred' del user - gc.collect() + gc_collect() assert len(s.identity_map) == 1 assert len(s.dirty) == 1 assert None not in s.dirty s.flush() - gc.collect() + gc_collect() assert not s.dirty assert not s.identity_map @@ -787,7 +788,7 @@ class SessionTest(_fixtures.FixtureTest): user = s.query(User).one() user = None print s.identity_map - gc.collect() + gc_collect() assert len(s.identity_map) == 1 user = s.query(User).one() @@ -810,7 +811,7 @@ class SessionTest(_fixtures.FixtureTest): self.assert_(len(s.identity_map) == 0) self.assert_(s.prune() == 0) s.flush() - gc.collect() + gc_collect() self.assert_(s.prune() == 9) self.assert_(len(s.identity_map) == 1) diff --git a/test/orm/transaction.py b/test/orm/transaction.py index 444a851dd1..c845c379f0 100644 --- a/test/orm/transaction.py +++ b/test/orm/transaction.py @@ -5,8 +5,10 @@ from sqlalchemy.orm import attributes from sqlalchemy import exc as sa_exc from sqlalchemy.orm import * from testlib import * +from testlib.compat import gc_collect from testlib.fixtures import * + class TransactionTest(FixtureTest): keep_mappers = True session = sessionmaker() @@ -104,7 +106,7 @@ class AutoExpireTest(TransactionTest): assert u1_state not in s.identity_map.all_states() assert u1_state not in s._deleted del u1 - gc.collect() + gc_collect() assert u1_state.obj() is None s.rollback() diff --git a/test/perf/masscreate.py b/test/perf/masscreate.py index 10f6514f0e..5b8e0da555 100644 --- a/test/perf/masscreate.py +++ b/test/perf/masscreate.py @@ -2,7 +2,6 @@ import testenv; testenv.simple_setup() from sqlalchemy.orm import attributes -from testlib import gc import time manage_attributes = True @@ -34,7 +33,6 @@ for i in range(0,130): attributes.manage(a) a.email = 'foo@bar.com' u.addresses.append(a) -# gc.collect() # print len(managed_attributes) # managed_attributes.clear() total = time.time() - now diff --git a/test/perf/masscreate2.py b/test/perf/masscreate2.py index 88a4d3c17e..cedd67e177 100644 --- a/test/perf/masscreate2.py +++ b/test/perf/masscreate2.py @@ -3,7 +3,7 @@ import testenv; testenv.simple_setup() import random, string from sqlalchemy.orm import attributes -from testlib import gc +from testlib.compat import gc_collect # with this test, run top. make sure the Python process doenst grow in size arbitrarily. @@ -33,4 +33,4 @@ for i in xrange(1000): a.user = u print "clearing" #managed_attributes.clear() - gc.collect() + gc_collect() diff --git a/test/perf/massload.py b/test/perf/massload.py index 4e2b05d123..ecc45e2b91 100644 --- a/test/perf/massload.py +++ b/test/perf/massload.py @@ -42,7 +42,7 @@ class LoadTest(TestBase, AssertsExecutionResults): query = sess.query(Item) for x in range (1,NUM/100): # this is not needed with cpython which clears non-circular refs immediately - #gc.collect() + #gc_collect() l = query.filter(items.c.item_id.between(x*100 - 100 + 1, x*100)).all() assert len(l) == 100 print "loaded ", len(l), " items " diff --git a/test/perf/masssave.py b/test/perf/masssave.py index bf65c8fdf7..5dc0f19965 100644 --- a/test/perf/masssave.py +++ b/test/perf/masssave.py @@ -1,4 +1,5 @@ import testenv; testenv.configure_for_tests() +import gc import types from sqlalchemy import * from sqlalchemy.orm import * diff --git a/test/perf/objselectspeed.py b/test/perf/objselectspeed.py index 788bdcd46b..0c2c6084dd 100644 --- a/test/perf/objselectspeed.py +++ b/test/perf/objselectspeed.py @@ -2,7 +2,7 @@ import testenv; testenv.simple_setup() import time, resource from sqlalchemy import * from sqlalchemy.orm import * -from testlib import gc +from testlib.compat import gc_collect db = create_engine('sqlite://') @@ -69,35 +69,35 @@ def all(): usage.snap = lambda stats=None: setattr( usage, 'last', stats or resource.getrusage(resource.RUSAGE_SELF)) - gc.collect() + gc_collect() usage.snap() t = time.clock() sqlite_select(RawPerson) t2 = time.clock() usage('sqlite select/native') - gc.collect() + gc_collect() usage.snap() t = time.clock() sqlite_select(Person) t2 = time.clock() usage('sqlite select/instrumented') - gc.collect() + gc_collect() usage.snap() t = time.clock() sql_select(RawPerson) t2 = time.clock() usage('sqlalchemy.sql select/native') - gc.collect() + gc_collect() usage.snap() t = time.clock() sql_select(Person) t2 = time.clock() usage('sqlalchemy.sql select/instrumented') - gc.collect() + gc_collect() usage.snap() t = time.clock() orm_select() diff --git a/test/perf/objupdatespeed.py b/test/perf/objupdatespeed.py index a6ed83a07d..785766f8bf 100644 --- a/test/perf/objupdatespeed.py +++ b/test/perf/objupdatespeed.py @@ -3,6 +3,8 @@ import time, resource from sqlalchemy import * from sqlalchemy.orm import * from testlib import * +from testlib.compat import gc_collect + NUM = 100 @@ -72,14 +74,14 @@ def all(): session = create_session() - gc.collect() + gc_collect() usage.snap() t = time.clock() people = orm_select(session) t2 = time.clock() usage('load objects') - gc.collect() + gc_collect() usage.snap() t = time.clock() update_and_flush(session, people) diff --git a/test/perf/sessions.py b/test/perf/sessions.py index 9d0e480f89..a9f948ec90 100644 --- a/test/perf/sessions.py +++ b/test/perf/sessions.py @@ -2,6 +2,7 @@ import testenv; testenv.configure_for_tests() from sqlalchemy import * from sqlalchemy.orm import * from testlib import * +from testlib.compat import gc_collect from testlib import fixtures # in this test we are specifically looking for time spent in the attributes.InstanceState.__cleanup() method. @@ -58,7 +59,7 @@ class SessionTest(TestBase, AssertsExecutionResults): sess.close() del sess - gc.collect() + gc_collect() @profiling.profiled('dirty', report=True) def test_session_dirty(self): @@ -72,11 +73,11 @@ class SessionTest(TestBase, AssertsExecutionResults): t2.c2 = 'this is some modified text' del t1s - gc.collect() + gc_collect() sess.close() del sess - gc.collect() + gc_collect() @profiling.profiled('noclose', report=True) def test_session_noclose(self): @@ -87,8 +88,7 @@ class SessionTest(TestBase, AssertsExecutionResults): t1s[index].t2s del sess - gc.collect() - + gc_collect() if __name__ == '__main__': diff --git a/test/profiling/memusage.py b/test/profiling/memusage.py index fb5df227d6..7b684f1a97 100644 --- a/test/profiling/memusage.py +++ b/test/profiling/memusage.py @@ -2,8 +2,10 @@ import testenv; testenv.configure_for_tests() from sqlalchemy.orm import mapper, relation, create_session, clear_mappers, sessionmaker from sqlalchemy.orm.mapper import _mapper_registry from sqlalchemy.orm.session import _sessions +import gc import operator -from testlib import gc, testing +from testlib import testing +from testlib.compat import gc_collect from testlib.sa import MetaData, Table, Column, Integer, String, ForeignKey, PickleType import sqlalchemy as sa from sqlalchemy.sql import column @@ -21,11 +23,11 @@ def profile_memory(func): # run the test 50 times. if length of gc.get_objects() # keeps growing, assert false def profile(*args): - gc.collect() + gc_collect() samples = [0 for x in range(0, 50)] for x in range(0, 50): func(*args) - gc.collect() + gc_collect() samples[x] = len(gc.get_objects()) print "sample gc sizes:", samples @@ -49,7 +51,7 @@ def profile_memory(func): def assert_no_mappers(): clear_mappers() - gc.collect() + gc_collect() assert len(_mapper_registry) == 0 class EnsureZeroed(_base.ORMTest): @@ -381,7 +383,7 @@ class MemUsageTest(EnsureZeroed): testing.eq_(len(session.identity_map._mutable_attrs), 12) testing.eq_(len(session.identity_map), 12) obj = None - gc.collect() + gc_collect() testing.eq_(len(session.identity_map._mutable_attrs), 0) testing.eq_(len(session.identity_map), 0) diff --git a/test/testlib/__init__.py b/test/testlib/__init__.py index 3996364a07..5dea60322b 100644 --- a/test/testlib/__init__.py +++ b/test/testlib/__init__.py @@ -17,7 +17,6 @@ from testlib.testing import \ from testlib.orm import mapper import testlib.profiling as profiling import testlib.engines as engines -import testlib.sa_gc as gc import testlib.requires as requires from testlib.compat import _function_named @@ -28,7 +27,7 @@ __all__ = ('testing', 'rowset', 'TestBase', 'AssertsExecutionResults', 'ORMTest', 'AssertsCompiledSQL', 'ComparesTables', - 'profiling', 'engines', 'gc', + 'profiling', 'engines', '_function_named') diff --git a/test/testlib/compat.py b/test/testlib/compat.py index dd23560be8..8f99527f93 100644 --- a/test/testlib/compat.py +++ b/test/testlib/compat.py @@ -1,4 +1,9 @@ -import sys, types, __builtin__ +import gc +import sys +import time +import types +import __builtin__ + __all__ = '_function_named', 'callable', 'py3k', 'jython' @@ -26,3 +31,12 @@ if py3k: else: callable = __builtin__.callable +if sys.platform.startswith('java'): + def gc_collect(*args): + gc.collect() + time.sleep(0.1) + gc.collect() + gc.collect() + return 0 +else: + gc_collect = gc.collect diff --git a/test/testlib/profiling.py b/test/testlib/profiling.py index 89db330111..41a47b00a3 100644 --- a/test/testlib/profiling.py +++ b/test/testlib/profiling.py @@ -1,7 +1,7 @@ """Profiling support for unit and performance tests.""" import os, sys -from testlib.compat import _function_named +from testlib.compat import _function_named, gc_collect import testlib.config __all__ = 'profiled', 'function_call_count', 'conditional_call_count' @@ -179,7 +179,7 @@ def _profile_cProfile(filename, fn, *args, **kw): import cProfile, gc, pstats, time load_stats = lambda: pstats.Stats(filename) - gc.collect() + gc_collect() began = time.time() cProfile.runctx('result = fn(*args, **kw)', globals(), locals(), @@ -192,7 +192,7 @@ def _profile_hotshot(filename, fn, *args, **kw): import gc, hotshot, hotshot.stats, time load_stats = lambda: hotshot.stats.load(filename) - gc.collect() + gc_collect() prof = hotshot.Profile(filename) began = time.time() prof.start() diff --git a/test/testlib/sa_gc.py b/test/testlib/sa_gc.py deleted file mode 100644 index 6e18575688..0000000000 --- a/test/testlib/sa_gc.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Cross platform garbage collection utility""" -import gc -import sys -import time - -if sys.platform.startswith('java'): - def collect(*args): - gc.collect() - time.sleep(0.1) - gc.collect() - gc.collect() - return 0 -else: - collect = gc.collect -- 2.47.3