From: Mike Bayer Date: Sat, 6 Jan 2007 20:32:47 +0000 (+0000) Subject: - added a mutex to the mapper compilation step. ive been reluctant to add any kind X-Git-Tag: rel_0_3_4~52 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1a5ba21076650f085b4583b0d9a862e611b69429;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - added a mutex to the mapper compilation step. ive been reluctant to add any kind of threading anything to SA but this is one spot that its its really needed since mappers are typically "global", and while their state does not change during normal operation, the initial compilation step does modify internal state significantly, and this step usually occurs not at module-level initialization time (unless you call compile()) but at first-request time - added "compile_mappers()" function as a shortcut to compiling all mappers --- diff --git a/CHANGES b/CHANGES index 423a47e7c8..81e5742080 100644 --- a/CHANGES +++ b/CHANGES @@ -1,31 +1,43 @@ -- added "BIGSERIAL" support for postgres table with PGBigInteger/autoincrement -- fix to MapperExtension create_instance so that entity_name properly associated +0.3.4 +- general: + - global "insure"->"ensure" change. in US english "insure" is actually + largely interchangeable with "ensure" (so says the dictionary), so I'm not + completely illiterate, but its definitely sub-optimal to "ensure" which is + non-ambiguous. +- sql: + - added "BIGSERIAL" support for postgres table with PGBigInteger/autoincrement + - fixes to postgres reflection to better handle when schema names are present; + thanks to jason (at) ncsmags.com [ticket:402] + - fix to correlation of subqueries when the column list of the select statement + is constructed with individual calls to append_column(); this fixes an ORM + bug whereby nested select statements were not getting correlated with the + main select generated by the Query object. + - default "timezone" setting is now False. this corresponds to Python's datetime + behavior as well as Postgres' timestamp/time types (which is the only timezone-sensitive + dialect at the moment) [ticket:414] +- firebird: + - order of constraint creation puts primary key first before all other constraints; + required for firebird, not a bad idea for others [ticket:408] + - Firebird fix to autoload multifield foreign keys [ticket:409] + - Firebird NUMERIC type properly handles a type without precision [ticket:409] +- orm: + - added a mutex to the mapper compilation step. ive been reluctant to add any kind + of threading anything to SA but this is one spot that its its really needed since mappers + are typically "global", and while their state does not change during normal operation, the + initial compilation step does modify internal state significantly, and this step usually + occurs not at module-level initialization time (unless you call compile()) but at first-request + time + - added "compile_mappers()" function as a shortcut to compiling all mappers + - fix to MapperExtension create_instance so that entity_name properly associated with new instance -- fixes to postgres reflection to better handle when schema names are present; -thanks to jason (at) ncsmags.com [ticket:402] -- fix to correlation of subqueries when the column list of the select statement -is constructed with individual calls to append_column(); this fixes an ORM -bug whereby nested select statements were not getting correlated with the -main select generated by the Query object. -- speed enhancements to ORM object instantiation, eager loading of rows -- global "insure"->"ensure" change. in US english "insure" is actually -largely interchangeable with "ensure" (so says the dictionary), so I'm not -completely illiterate, but its definitely sub-optimal to "ensure" which is -non-ambiguous. -- invalid options sent to 'cascade' string will raise an exception [ticket:406] -- fixed bug in mapper refresh/expire whereby eager loaders didnt properly re-populate -item lists [ticket:407] -- order of constraint creation puts primary key first before all other constraints; -required for firebird, not a bad idea for others [ticket:408] -- Firebird fix to autoload multifield foreign keys [ticket:409] -- Firebird NUMERIC type properly handles a type without precision [ticket:409] -- fix to post_update to insure rows are updated even for non insert/delete scenarios -[ticket:413] -- default "timezone" setting is now False. this corresponds to Python's datetime -behavior as well as Postgres' timestamp/time types (which is the only timezone-sensitive -dialect at the moment) [ticket:414] -- added an error message if you actually try to modify primary key values on an entity -and then flush it. + - speed enhancements to ORM object instantiation, eager loading of rows + - invalid options sent to 'cascade' string will raise an exception [ticket:406] + - fixed bug in mapper refresh/expire whereby eager loaders didnt properly re-populate + item lists [ticket:407] + - fix to post_update to insure rows are updated even for non insert/delete scenarios + [ticket:413] + - added an error message if you actually try to modify primary key values on an entity + and then flush it. 0.3.3 - string-based FROM clauses fixed, i.e. select(..., from_obj=["sometext"]) diff --git a/lib/sqlalchemy/orm/__init__.py b/lib/sqlalchemy/orm/__init__.py index d71b88d9cf..1569b02deb 100644 --- a/lib/sqlalchemy/orm/__init__.py +++ b/lib/sqlalchemy/orm/__init__.py @@ -18,7 +18,7 @@ from sqlalchemy.orm.session import Session as create_session from sqlalchemy.orm.session import object_session __all__ = ['relation', 'backref', 'eagerload', 'lazyload', 'noload', 'deferred', 'defer', 'undefer', 'extension', - 'mapper', 'clear_mappers', 'clear_mapper', 'class_mapper', 'object_mapper', 'MapperExtension', 'Query', + 'mapper', 'clear_mappers', 'compile_mappers', 'clear_mapper', 'class_mapper', 'object_mapper', 'MapperExtension', 'Query', 'cascade_mappers', 'polymorphic_union', 'create_session', 'synonym', 'contains_eager', 'EXT_PASS', 'object_session' ] @@ -59,7 +59,15 @@ def synonym(name, proxy=False): Used with the 'properties' dictionary sent to mapper().""" return properties.SynonymProperty(name, proxy=proxy) - + +def compile_mappers(): + """compile all mappers that have been defined. + + this is equivalent to calling compile() on any individual mapper.""" + if not len(mapper_registry): + return + mapper_registry.values()[0].compile() + def clear_mappers(): """remove all mappers that have been created thus far. diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index b6ff096169..3b1bb1dad9 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -26,6 +26,9 @@ NO_ATTRIBUTE = object() # returned by a MapperExtension method to indicate a "do nothing" response EXT_PASS = object() + +# lock used to synchronize the "mapper compile" step +_COMPILE_MUTEX = util.threading.Lock() class Mapper(object): """Defines the correlation of class attributes to database table columns. @@ -248,16 +251,22 @@ class Mapper(object): this is the 'external' version of the method which is not reentrant.""" if self.__is_compiled: return self + _COMPILE_MUTEX.acquire() + try: + # double-check inside mutex + if self.__is_compiled: + return self + self._compile_all() - self._compile_all() - - # if we're not primary, compile us - if self.non_primary: - self._do_compile() - self._initialize_properties() + # if we're not primary, compile us + if self.non_primary: + self._do_compile() + self._initialize_properties() - return self - + return self + finally: + _COMPILE_MUTEX.release() + def _compile_all(self): # compile all primary mappers for mapper in mapper_registry.values(): diff --git a/test/perf/threaded_compile.py b/test/perf/threaded_compile.py new file mode 100644 index 0000000000..3c521f2316 --- /dev/null +++ b/test/perf/threaded_compile.py @@ -0,0 +1,70 @@ +# tests the COMPILE_MUTEX in mapper compilation + +from sqlalchemy import * +import thread, time, random +from sqlalchemy.orm import mapperlib + +meta = BoundMetaData('sqlite:///foo.db') + +t1 = Table('t1', meta, + Column('c1', Integer, primary_key=True), + Column('c2', String(30)) + ) + +t2 = Table('t2', meta, + Column('c1', Integer, primary_key=True), + Column('c2', String(30)), + Column('t1c1', None, ForeignKey('t1.c1')) +) +meta.create_all() + +class T1(object): + pass + +class T2(object): + pass + +class FakeLock(object): + def acquire(self):pass + def release(self):pass + +# uncomment this to disable the mutex in mapper compilation; +# should produce thread collisions +#mapperlib._COMPILE_MUTEX = FakeLock() + +existing_compile_all = mapperlib.Mapper._compile_all +state = [False] +# decorate mapper's _compile_all() method; the mutex in mapper.compile() +# should insure that this method is only called once by a single thread only +def monkeypatch_compile_all(self): + if state[0]: + raise "thread collision" + state[0] = True + try: + print "compile", thread.get_ident() + time.sleep(1 + random.random()) + existing_compile_all(self) + finally: + state[0] = False +mapperlib.Mapper._compile_all = monkeypatch_compile_all + +def run1(): + print "T1", thread.get_ident() + class_mapper(T1) + +def run2(): + print "T2", thread.get_ident() + class_mapper(T2) + +for i in range(0,1): + clear_mappers() + mapper(T1, t1, properties={'t2':relation(T2, backref="t1")}) + mapper(T2, t2) + #compile_mappers() + print "START" + for j in range(0, 5): + thread.start_new_thread(run1, ()) + thread.start_new_thread(run2, ()) + print "WAIT" + time.sleep(5) +