From: Mike Bayer Date: Wed, 21 Jul 2010 20:19:08 +0000 (-0400) Subject: - The beaker_caching example has been reorgnized X-Git-Tag: rel_0_6_4~80 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5ab558004c816deb258eef16b340c1cf9cda7776;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - The beaker_caching example has been reorgnized such that the Session, cache manager, declarative_base are part of environment, and custom cache code is portable and now within "caching_query.py". This allows the example to be easier to "drop in" to existing projects. --- diff --git a/CHANGES b/CHANGES index 3aaff9b001..9662d86f96 100644 --- a/CHANGES +++ b/CHANGES @@ -17,6 +17,14 @@ CHANGES indexes on columns with similar names still have unique names. [ticket:1855] +- examples + - The beaker_caching example has been reorgnized + such that the Session, cache manager, + declarative_base are part of environment, and + custom cache code is portable and now within + "caching_query.py". This allows the example to + be easier to "drop in" to existing projects. + 0.6.3 ===== - orm diff --git a/examples/beaker_caching/__init__.py b/examples/beaker_caching/__init__.py index a4e8ec1f13..d7e97efea2 100644 --- a/examples/beaker_caching/__init__.py +++ b/examples/beaker_caching/__init__.py @@ -48,12 +48,13 @@ The demo scripts themselves, in order of complexity, are run as follows:: Listing of files: - environment.py - Establish data / cache file paths, and configurations, + environment.py - Establish the Session, the Beaker cache + manager, data / cache file paths, and configurations, bootstrap fixture data if necessary. - meta.py - Represent persistence structures which allow the usage of - Beaker caching with SQLAlchemy. Introduces a query option called - FromCache. + caching_query.py - Represent functions and classes + which allow the usage of Beaker caching with SQLAlchemy. + Introduces a query option called FromCache. model.py - The datamodel, which represents Person that has multiple Address objects, each with PostalCode, City, Country diff --git a/examples/beaker_caching/advanced.py b/examples/beaker_caching/advanced.py index e14b969738..1130600336 100644 --- a/examples/beaker_caching/advanced.py +++ b/examples/beaker_caching/advanced.py @@ -6,9 +6,9 @@ and collection caching. """ -import environment +from environment import Session from model import Person, Address, cache_address_bits -from meta import Session, FromCache, RelationshipCache +from caching_query import FromCache, RelationshipCache from sqlalchemy.orm import joinedload def load_name_range(start, end, invalidate=False): diff --git a/examples/beaker_caching/environment.py b/examples/beaker_caching/environment.py index cdf1794fd4..ea946606ca 100644 --- a/examples/beaker_caching/environment.py +++ b/examples/beaker_caching/environment.py @@ -4,10 +4,29 @@ Establish data / cache file paths, and configurations, bootstrap fixture data if necessary. """ -import meta, model, fixture_data +import caching_query from sqlalchemy import create_engine +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from beaker import cache import os +# Beaker CacheManager. A home base for cache configurations. +cache_manager = cache.CacheManager() + +# scoped_session. Apply our custom CachingQuery class to it, +# using a callable that will associate the cache_manager +# with the Query. +Session = scoped_session( + sessionmaker( + query_cls=caching_query.query_callable(cache_manager) + ) + ) + +# global declarative base class. +Base = declarative_base() + + root = "./beaker_data/" if not os.path.exists(root): @@ -19,10 +38,10 @@ if not os.path.exists(root): dbfile = os.path.join(root, "beaker_demo.db") engine = create_engine('sqlite:///%s' % dbfile, echo=True) -meta.Session.configure(bind=engine) +Session.configure(bind=engine) # configure the "default" cache region. -meta.cache_manager.regions['default'] ={ +cache_manager.regions['default'] ={ # using type 'file' to illustrate # serialized persistence. In reality, @@ -39,6 +58,10 @@ meta.cache_manager.regions['default'] ={ } installed = False -if not os.path.exists(dbfile): - fixture_data.install() - installed = True \ No newline at end of file + +def bootstrap(): + global installed + import fixture_data + if not os.path.exists(dbfile): + fixture_data.install() + installed = True \ No newline at end of file diff --git a/examples/beaker_caching/fixture_data.py b/examples/beaker_caching/fixture_data.py index 258d1ae86c..67af746e38 100644 --- a/examples/beaker_caching/fixture_data.py +++ b/examples/beaker_caching/fixture_data.py @@ -5,7 +5,7 @@ Canadian cities. Then, 100 Person records are installed, each with a randomly selected postal code. """ -from meta import Session, Base +from environment import Session, Base from model import City, Country, PostalCode, Person, Address import random diff --git a/examples/beaker_caching/helloworld.py b/examples/beaker_caching/helloworld.py index 3d37777d7c..f64fcdd2e4 100644 --- a/examples/beaker_caching/helloworld.py +++ b/examples/beaker_caching/helloworld.py @@ -4,9 +4,9 @@ Illustrate how to load some data, and cache the results. """ -import environment +from environment import Session from model import Person -from meta import Session, FromCache +from caching_query import FromCache # load Person objects. cache the result under the namespace "all_people". print "loading people...." diff --git a/examples/beaker_caching/local_session_caching.py b/examples/beaker_caching/local_session_caching.py index 0480d838ed..cce2835f1e 100644 --- a/examples/beaker_caching/local_session_caching.py +++ b/examples/beaker_caching/local_session_caching.py @@ -53,22 +53,22 @@ class ScopedSessionNamespace(container.MemoryNamespaceManager): if __name__ == '__main__': - import environment - import meta + from environment import Session, cache_manager + from caching_query import FromCache # create a Beaker container type called "ext:local_session". # it will reference the ScopedSession in meta. - ScopedSessionNamespace.create_session_container("ext:local_session", meta.Session) + ScopedSessionNamespace.create_session_container("ext:local_session", Session) # set up a region based on this new container type. - meta.cache_manager.regions['local_session'] ={'type':'ext:local_session'} + cache_manager.regions['local_session'] ={'type':'ext:local_session'} from model import Person # query to load Person by name, with criterion # of "person 10" - q = meta.Session.query(Person).\ - options(meta.FromCache("local_session", "by_name")).\ + q = Session.query(Person).\ + options(FromCache("local_session", "by_name")).\ filter(Person.name=="person 10") # load from DB @@ -79,7 +79,7 @@ if __name__ == '__main__': # clear out the Session. The "_beaker_cache" dictionary # disappears with it. - meta.Session.remove() + Session.remove() # query calls from DB again person10 = q.one() @@ -91,6 +91,6 @@ if __name__ == '__main__': # that would change the results of a cached query, such as # inserts, deletes, or modification to attributes that are # part of query criterion, still require careful invalidation. - from meta import _get_cache_parameters + from caching_query import _get_cache_parameters cache, key = _get_cache_parameters(q) assert person10 is cache.get(key)[0] diff --git a/examples/beaker_caching/meta.py b/examples/beaker_caching/meta.py deleted file mode 100644 index a4e43caf19..0000000000 --- a/examples/beaker_caching/meta.py +++ /dev/null @@ -1,253 +0,0 @@ -"""meta.py - -Represent persistence structures which allow the usage of -Beaker caching with SQLAlchemy. - -The three new concepts introduced here are: - - * CachingQuery - a Query subclass that caches and - retrieves results in/from Beaker. - * FromCache - a query option that establishes caching - parameters on a Query - * RelationshipCache - a variant of FromCache which is specific - to a query invoked during a lazy load. - * _params_from_query - extracts value parameters from - a Query. - -The rest of what's here are standard SQLAlchemy and -Beaker constructs. - -""" -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.orm.interfaces import MapperOption -from sqlalchemy.orm.query import Query -from sqlalchemy.sql import visitors -from sqlalchemy.ext.declarative import declarative_base -from beaker import cache - -class CachingQuery(Query): - """A Query subclass which optionally loads full results from a Beaker - cache region. - - The CachingQuery is instructed to load from cache based on two optional - attributes configured on the instance, called 'cache_region' and 'cache_namespace'. - - When these attributes are present, any iteration of the Query will configure - a Beaker cache against this region and a generated namespace, which takes - into account the 'cache_namespace' name as well as the entities this query - is created against (i.e. the columns and classes sent to the constructor). - The 'cache_namespace' is a string name that represents a particular structure - of query. E.g. a query that filters on a name might use the name "by_name", - a query that filters on a date range to a joined table might use the name - "related_date_range". - - The Query then attempts to retrieved a cached value using a key, which - is generated from all the parameterized values present in the Query. In - this way, the combination of "cache_namespace" and embedded parameter values - correspond exactly to the lexical structure of a SQL statement combined - with its bind parameters. If no such key exists then the ultimate SQL - is emitted and the objects loaded. - - The returned objects, if loaded from cache, are merged into the Query's - session using Session.merge(load=False), which is a fast performing - method to ensure state is present. - - The FromCache and RelationshipCache mapper options below represent - the "public" method of - configuring the "cache_region" and "cache_namespace" attributes. - RelationshipCache has the ability to be invoked upon lazy loaders embedded - in an object graph. - - """ - - def __iter__(self): - """override __iter__ to pull results from Beaker - if particular attributes have been configured. - """ - if hasattr(self, '_cache_parameters'): - cache, cache_key = _get_cache_parameters(self) - ret = cache.get_value(cache_key, createfunc=lambda: list(Query.__iter__(self))) - - # merge the result in. - return self.merge_result(ret, load=False) - else: - return Query.__iter__(self) - - def invalidate(self): - """Invalidate the cache represented in this Query.""" - - cache, cache_key = _get_cache_parameters(self) - cache.remove(cache_key) - - def set_value(self, value): - """Set the value in the cache for this query.""" - - cache, cache_key = _get_cache_parameters(self) - cache.put(cache_key, value) - -def _get_cache_parameters(query): - """For a query with cache_region and cache_namespace configured, - return the correspoinding Cache instance and cache key, based - on this query's current criterion and parameter values. - - """ - if not hasattr(query, '_cache_parameters'): - raise ValueError("This Query does not have caching parameters configured.") - - region, namespace, cache_key = query._cache_parameters - - # cache namespace - the token handed in by the - # option + class we're querying against - namespace = " ".join([namespace] + [str(x) for x in query._entities]) - - # memcached wants this - namespace = namespace.replace(' ', '_') - - if cache_key is None: - # cache key - the value arguments from this query's parameters. - args = _params_from_query(query) - cache_key = " ".join([str(x) for x in args]) - - # get cache - cache = cache_manager.get_cache_region(namespace, region) - - # optional - hash the cache_key too for consistent length - # import uuid - # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) - - return cache, cache_key - -def _set_cache_parameters(query, region, namespace, cache_key): - - if hasattr(query, '_cache_parameters'): - region, namespace, cache_key = query._cache_parameters - raise ValueError("This query is already configured " - "for region %r namespace %r" % - (region, namespace) - ) - query._cache_parameters = region, namespace, cache_key - -class FromCache(MapperOption): - """Specifies that a Query should load results from a cache.""" - - propagate_to_loaders = False - - def __init__(self, region, namespace, cache_key=None): - """Construct a new FromCache. - - :param region: the cache region. Should be a - region configured in the Beaker CacheManager. - - :param namespace: the cache namespace. Should - be a name uniquely describing the target Query's - lexical structure. - - :param cache_key: optional. A string cache key - that will serve as the key to the query. Use this - if your query has a huge amount of parameters (such - as when using in_()) which correspond more simply to - some other identifier. - - """ - self.region = region - self.namespace = namespace - self.cache_key = cache_key - - def process_query(self, query): - """Process a Query during normal loading operation.""" - - _set_cache_parameters(query, self.region, self.namespace, self.cache_key) - -class RelationshipCache(MapperOption): - """Specifies that a Query as called within a "lazy load" - should load results from a cache.""" - - propagate_to_loaders = True - - def __init__(self, region, namespace, attribute): - """Construct a new RelationshipCache. - - :param region: the cache region. Should be a - region configured in the Beaker CacheManager. - - :param namespace: the cache namespace. Should - be a name uniquely describing the target Query's - lexical structure. - - :param attribute: A Class.attribute which - indicates a particular class relationship() whose - lazy loader should be pulled from the cache. - - """ - self.region = region - self.namespace = namespace - self._relationship_options = { - ( attribute.property.parent.class_, attribute.property.key ) : self - } - - def process_query_conditionally(self, query): - """Process a Query that is used within a lazy loader. - - (the process_query_conditionally() method is a SQLAlchemy - hook invoked only within lazyload.) - - """ - if query._current_path: - mapper, key = query._current_path[-2:] - - for cls in mapper.class_.__mro__: - if (cls, key) in self._relationship_options: - relationship_option = self._relationship_options[(cls, key)] - _set_cache_parameters( - query, - relationship_option.region, - relationship_option.namespace, - None) - - def and_(self, option): - """Chain another RelationshipCache option to this one. - - While many RelationshipCache objects can be specified on a single - Query separately, chaining them together allows for a more efficient - lookup during load. - - """ - self._relationship_options.update(option._relationship_options) - return self - - -def _params_from_query(query): - """Pull the bind parameter values from a query. - - This takes into account any scalar attribute bindparam set up. - - E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) - would return [5, 7]. - - """ - v = [] - def visit_bindparam(bind): - value = query._params.get(bind.key, bind.value) - - # lazyloader may dig a callable in here, intended - # to late-evaluate params after autoflush is called. - # convert to a scalar value. - if callable(value): - value = value() - - v.append(value) - if query._criterion is not None: - visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) - return v - -# Beaker CacheManager. A home base for cache configurations. -# Configured at startup in __init__.py -cache_manager = cache.CacheManager() - -# global application session. -# configured at startup in __init__.py -Session = scoped_session(sessionmaker(query_cls=CachingQuery)) - -# global declarative base class. -Base = declarative_base() - diff --git a/examples/beaker_caching/model.py b/examples/beaker_caching/model.py index 6ea5e79044..c139d3284c 100644 --- a/examples/beaker_caching/model.py +++ b/examples/beaker_caching/model.py @@ -10,7 +10,8 @@ City --(has a)--> Country """ from sqlalchemy import Column, Integer, String, ForeignKey from sqlalchemy.orm import relationship -from meta import Base, FromCache, Session, RelationshipCache +from caching_query import FromCache, RelationshipCache +from environment import Base, bootstrap class Country(Base): __tablename__ = 'country' @@ -102,3 +103,4 @@ cache_address_bits = RelationshipCache("default", "byid", PostalCode.city).\ RelationshipCache("default", "byid", Address.postal_code) ) +bootstrap() \ No newline at end of file diff --git a/examples/beaker_caching/relation_caching.py b/examples/beaker_caching/relation_caching.py index 100f5412d5..1691b071b6 100644 --- a/examples/beaker_caching/relation_caching.py +++ b/examples/beaker_caching/relation_caching.py @@ -5,9 +5,8 @@ related PostalCode, City, Country objects should be pulled from long term cache. """ -import environment +from environment import Session, root from model import Person, Address, cache_address_bits -from meta import Session from sqlalchemy.orm import joinedload import os @@ -22,4 +21,4 @@ print "\n\nIf this was the first run of relationship_caching.py, SQL was likely "To clear the cache, delete the directory %r. \n"\ "This will cause a re-load of cities, postal codes and countries on "\ "the next run.\n"\ - % os.path.join(environment.root, 'container_file') + % os.path.join(root, 'container_file')