From: Mike Bayer Date: Sat, 1 Sep 2007 21:21:29 +0000 (+0000) Subject: - got all examples working X-Git-Tag: rel_0_4beta6~64 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=5df1759e151274594ca4691419f6be7e91257635;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - got all examples working - inline default execution occurs for *all* non-PK columns unconditionally - preexecute only for non-executemany PK cols on PG, Oracle, etc. - new default docs --- diff --git a/CHANGES b/CHANGES index 842d513f3b..4c197522f1 100644 --- a/CHANGES +++ b/CHANGES @@ -23,10 +23,12 @@ CHANGES - Fixed OrderedProperties pickling [ticket:762] -- Defaults and sequences now execute "inline" for all executemany() - calls, using no prefetch whatsoever. inline=True flag on any - insert/update statement also forces the same behavior with a single - execute(). +- SQL-expression defaults and sequences now execute "inline" for all non-primary key + columns during an INSERT or UPDATE, and for all columns during an executemany()-style + call. inline=True flag on any insert/update statement also forces the same + behavior with a single execute(). result.postfetch_cols() is a collection of columns + for which the previous single insert or update statement contained a SQL-side + default expression. - fixed PG executemany() behavior, [ticket:759] diff --git a/doc/build/content/metadata.txt b/doc/build/content/metadata.txt index 068d1af9f0..1efe8607ab 100644 --- a/doc/build/content/metadata.txt +++ b/doc/build/content/metadata.txt @@ -295,16 +295,16 @@ Entire groups of Tables can be created and dropped directly from the `MetaData` pref_value VARCHAR(100) ) -### Column Defaults and OnUpdates {@name=defaults} +### Column Insert/Update Defaults {@name=defaults} -SQLAlchemy includes flexible constructs in which to create default values for columns upon the insertion of rows, as well as upon update. These defaults can take several forms: a constant, a Python callable to be pre-executed before the SQL is executed, a SQL expression or function to be pre-executed before the SQL is executed, a pre-executed Sequence (for databases that support sequences), or a "passive" default, which is a default function triggered by the database itself upon insert, the value of which can then be post-fetched by the engine, provided the row provides a primary key in which to call upon. +SQLAlchemy includes several constructs which provide default values provided during INSERT and UPDATE statements. The defaults may be provided as Python constants, Python functions, or SQL expressions, and the SQL expressions themselves may be "pre-executed", executed inline within the insert/update statement itself, or can be created as a SQL level "default" placed on the table definition itself. A "default" value by definition is only invoked if no explicit value is passed into the INSERT or UPDATE statement. -#### Pre-Executed Insert Defaults {@name=oninsert} +#### Pre-Executed Python Functions {@name=preexecute_functions} -A basic default is most easily specified by the "default" keyword argument to Column. This defines a value, function, or SQL expression that will be pre-executed to produce the new value, before the row is inserted: +The "default" keyword argument on Column can reference a Python value or callable which is invoked at the time of an insert: {python} - # a function to create primary key ids + # a function which counts upwards i = 0 def mydefault(): global i @@ -318,46 +318,48 @@ A basic default is most easily specified by the "default" keyword argument to Co # a scalar default Column('key', String(10), default="default") ) - -The "default" keyword can also take SQL expressions, including select statements or direct function calls: + +Similarly, the "onupdate" keyword does the same thing for update statements: {python} + import datetime + t = Table("mytable", meta, Column('id', Integer, primary_key=True), - # define 'create_date' to default to now() - Column('create_date', DateTime, default=func.now()), - - # define 'key' to pull its default from the 'keyvalues' table - Column('key', String(20), default=keyvalues.select(keyvalues.c.type='type1', limit=1)) - ) - -The "default" keyword argument is shorthand for using a ColumnDefault object in a column definition. This syntax is optional, but is required for other types of defaults, futher described below: - - {python} - Column('mycolumn', String(30), ColumnDefault(func.get_data())) + # define 'last_updated' to be populated with datetime.now() + Column('last_updated', DateTime, onupdate=datetime.now), + ) -#### Pre-Executed OnUpdate Defaults {@name=onupdate} +#### Pre-executed and Inline SQL Expressions {@name=sqlexpression} -Similar to an on-insert default is an on-update default, which is most easily specified by the "onupdate" keyword to Column, which also can be a constant, plain Python function or SQL expression: +The "default" and "onupdate" keywords may also be passed SQL expressions, including select statements or direct function calls: {python} t = Table("mytable", meta, Column('id', Integer, primary_key=True), - - # define 'last_updated' to be populated with current_timestamp (the ANSI-SQL version of now()) - Column('last_updated', DateTime, onupdate=func.current_timestamp()), - ) + # define 'create_date' to default to now() + Column('create_date', DateTime, default=func.now()), + + # define 'key' to pull its default from the 'keyvalues' table + Column('key', String(20), default=keyvalues.select(keyvalues.c.type='type1', limit=1)) + + # define 'last_modified' to use the current_timestamp SQL function on update + Column('last_modified', DateTime, onupdate=func.current_timestamp()) + ) -To use an explicit ColumnDefault object to specify an on-update, use the "for_update" keyword argument: +The above SQL functions are always executed "inline" with the INSERT or UPDATE statement being executed, **unless** several conditions are met: + * the column is a primary key column + * the database dialect does not support a usable `cursor.lastrowid` accessor (or equivalent); this currently includes Postgres, Oracle, and Firebird. + * the statement is a single execution, i.e. only supplies one set of parameters and doesn't use "executemany" behavior + * the `inline=True` flag is not set on the `Insert()` or `Update()` construct. - {python} - Column('mycolumn', String(30), ColumnDefault(func.get_data(), for_update=True)) - -#### Inline Default Execution: PassiveDefault {@name=passive} +For a statement which executes with `inline=False` and is not an executemany execution, the returned `ResultProxy` will contain a collection accessible via `result.postfetch_cols()`, which contains a list of all `Column` objects which had an inline-executed default. Similarly, all parameters which were bound to the statement, including all Python and SQL expressions which were pre-executed, are present in the `last_inserted_params()` or `last_updated_params()` collections on `ResultProxy`. The `last_inserted_ids()` collection contains a list of primary key values for the row inserted. + +#### DDL-Level Defaults {@name=passive} -A PassiveDefault indicates an column default that is executed upon INSERT by the database. This construct is used to specify a SQL function that will be specified as "DEFAULT" when creating tables. +A variant on a SQL expression default is the `PassiveDefault`, which gets placed in the CREATE TABLE statement during a `create()` operation: {python} t = Table('test', meta, @@ -371,31 +373,7 @@ A create call for the above table will produce: mycolumn datetime default sysdate ) -PassiveDefault also sends a message to the `Engine` that data is available after an insert. The object-relational mapper system uses this information to post-fetch rows after the insert, so that instances can be refreshed with the new data. Below is a simplified version: - - {python} - # table with passive defaults - mytable = Table('mytable', engine, - Column('my_id', Integer, primary_key=True), - - # an on-insert database-side default - Column('data1', Integer, PassiveDefault("d1_func()")), - ) - # insert a row - r = mytable.insert().execute(name='fred') - - # check the result: were there defaults fired off on that row ? - if r.lastrow_has_defaults(): - # postfetch the row based on primary key. - # this only works for a table with primary key columns defined - primary_key = r.last_inserted_ids() - row = table.select(table.c.id == primary_key[0]) - -When Tables are reflected from the database using `autoload=True`, any DEFAULT values set on the columns will be reflected in the Table object as PassiveDefault instances. - -##### The Catch: Postgres Primary Key Defaults always Pre-Execute {@name=postgres} - -Current Postgres support does not rely upon OID's to determine the identity of a row. This is because the usage of OIDs has been deprecated with Postgres and they are disabled by default for table creates as of PG version 8. Pyscopg2's "cursor.lastrowid" function only returns OIDs. Therefore, when inserting a new row which has passive defaults set on the primary key columns, the default function is still pre-executed since SQLAlchemy would otherwise have no way of retrieving the row just inserted. +The behavior of `PassiveDefault` is similar to that of the regular default; if it's placed on a primary key column for a database which doesn't have a way to "postfetch" the ID, and the statement is not "inlined", the SQL expression is pre-executed; otherwise, SQLAlchemy lets the default fire off on the database side normally. #### Defining Sequences {@name=sequences} @@ -408,11 +386,11 @@ A table with a sequence looks like: Column("createdate", DateTime()) ) -The Sequence is used with Postgres or Oracle to indicate the name of a database sequence that will be used to create default values for a column. When a table with a Sequence on a column is created in the database by SQLAlchemy, the database sequence object is also created. Similarly, the database sequence is dropped when the table is dropped. Sequences are typically used with primary key columns. When using Postgres, if an integer primary key column defines no explicit Sequence or other default method, SQLAlchemy will create the column with the SERIAL keyword, and will pre-execute a sequence named "tablename_columnname_seq" in order to retrieve new primary key values, if they were not otherwise explicitly stated. Oracle, which has no "auto-increment" keyword, requires that a Sequence be specified for a table if automatic primary key generation is desired. +The `Sequence` object works a lot like the `default` keyword on `Column`, except that it only takes effect on a database which supports sequences. The same rules for pre- and inline execution apply. -A Sequence object can be defined on a Table that is then also used with a non-sequence-supporting database. In that case, the Sequence object is simply ignored. Note that a Sequence object is **entirely optional for all databases except Oracle**, as other databases offer options for auto-creating primary key values, such as AUTOINCREMENT, SERIAL, etc. SQLAlchemy will use these default methods for creating primary key values if no Sequence is present on the table metadata. +When the `Sequence` is associated with a table, CREATE and DROP statements issued for that table will also issue CREATE/DROP for the sequence object as well, thus "bundling" the sequence object with its parent table. -A sequence can also be specified with `optional=True` which indicates the Sequence should only be used on a database that requires an explicit sequence, and not those that supply some other method of providing integer values. At the moment, it essentially means "use this sequence only with Oracle and not Postgres". +The flag `optional=True` on `Sequence` will produce a sequence that is only used on databases which have no "autoincrementing" capability. For example, Postgres supports primary key generation using the SERIAL keyword, whereas Oracle has no such capability. Therefore, a `Sequence` placed on a primary key column with `optional=True` will only be used with an Oracle backend but not Postgres. ### Defining Constraints and Indexes {@name=constraints} diff --git a/examples/adjacencytree/basic_tree.py b/examples/adjacencytree/basic_tree.py index c6f49ccaea..9048ed8dc8 100644 --- a/examples/adjacencytree/basic_tree.py +++ b/examples/adjacencytree/basic_tree.py @@ -43,7 +43,11 @@ mapper(TreeNode, trees, properties=dict( id=trees.c.node_id, name=trees.c.node_name, parent_id=trees.c.parent_node_id, - children=relation(TreeNode, cascade="all", backref=backref("parent", remote_side=[trees.c.node_id]), collection_class=attribute_mapped_collection('name')), + children=relation(TreeNode, cascade="all", + backref=backref("parent", remote_side=[trees.c.node_id]), collection_class=attribute_mapped_collection('name'), + lazy=False, + join_depth=3 + ), )) print "\n\n\n----------------------------" diff --git a/examples/adjacencytree/byroot_tree.py b/examples/adjacencytree/byroot_tree.py index e6e57b5aa1..76039526b3 100644 --- a/examples/adjacencytree/byroot_tree.py +++ b/examples/adjacencytree/byroot_tree.py @@ -10,7 +10,7 @@ engine = create_engine('sqlite:///:memory:', echo=True) metadata = MetaData(engine) -"""create the treenodes table. This is ia basic adjacency list model table. +"""create the treenodes table. This is a basic adjacency list model table. One additional column, "root_node_id", references a "root node" row and is used in the 'byroot_tree' example.""" @@ -83,7 +83,6 @@ class TreeLoader(MapperExtension): append root nodes to the result list, and will attach child nodes to their appropriate parent node as they arrive from the select results. This allows a SELECT statement which returns both root and child nodes in one query to return a list of "roots".""" - isnew = flags.get('isnew', False) if instance.parent_id is None: @@ -108,14 +107,19 @@ print "----------------------------" metadata.create_all() -# the mapper is created with properties that specify "lazy=None" - this is because we are going -# to handle our own "eager load" of nodes based on root id mapper(TreeNode, trees, properties=dict( id=trees.c.node_id, name=trees.c.node_name, parent_id=trees.c.parent_node_id, root_id=trees.c.root_node_id, - root=relation(TreeNode, primaryjoin=trees.c.root_node_id==trees.c.node_id, remote_side=trees.c.node_id, lazy=None), + + # 'root' attribute. has a load-only backref '_descendants' that loads all nodes with the same root ID eagerly, + # which are intercepted by the TreeLoader extension and populated into the "children" collection. + root=relation(TreeNode, primaryjoin=trees.c.root_node_id==trees.c.node_id, remote_side=trees.c.node_id, lazy=None, + backref=backref('_descendants', lazy=False, join_depth=1, primaryjoin=trees.c.root_node_id==trees.c.node_id,viewonly=True)), + + # 'children' attribute. collection of immediate child nodes. this is a non-loading relation + # which is populated by the TreeLoader extension. children=relation(TreeNode, primaryjoin=trees.c.parent_node_id==trees.c.node_id, lazy=None, @@ -123,6 +127,8 @@ mapper(TreeNode, trees, properties=dict( collection_class=attribute_mapped_collection('name'), backref=backref('parent', primaryjoin=trees.c.parent_node_id==trees.c.node_id, remote_side=trees.c.node_id) ), + + # 'data' attribute. A collection of secondary objects which also loads eagerly. data=relation(TreeData, cascade="all, delete-orphan", lazy=False) ), extension = TreeLoader()) diff --git a/examples/association/proxied_association.py b/examples/association/proxied_association.py index 2dd60158b9..f7dd45c4ae 100644 --- a/examples/association/proxied_association.py +++ b/examples/association/proxied_association.py @@ -2,6 +2,7 @@ the usage of the associationproxy extension.""" from sqlalchemy import * +from sqlalchemy.orm import * from sqlalchemy.ext.selectresults import SelectResults from sqlalchemy.ext.associationproxy import AssociationProxy from datetime import datetime @@ -66,7 +67,7 @@ session.flush() # function to return items def item(name): - return session.query(Item).get_by(description=name) + return session.query(Item).filter_by(description=name).one() # create an order order = Order('john smith') @@ -88,7 +89,7 @@ session.flush() session.clear() # query the order, print items -order = session.query(Order).get_by(customer_name='john smith') +order = session.query(Order).filter_by(customer_name='john smith').one() # print items based on the OrderItem collection directly print [(item.item.description, item.price) for item in order.itemassociations] @@ -97,11 +98,11 @@ print [(item.item.description, item.price) for item in order.itemassociations] print [(item.description, item.price) for item in order.items] # print customers who bought 'MySQL Crowbar' on sale -result = session.query(Order).join('item').filter(and_(items.c.description=='MySQL Crowbar', items.c.price>orderitems.c.price)) +result = session.query(Order).join(['itemassociations', 'item']).filter(and_(Item.description=='MySQL Crowbar', Item.price>OrderItem.price)) print [order.customer_name for order in result] # print customers who got the special T-shirt discount -result = session.query(Order).join('item').filter(and_(items.c.description=='SA T-Shirt', items.c.price>orderitems.c.price)) +result = session.query(Order).join(['itemassociations', 'item']).filter(and_(Item.description=='SA T-Shirt', Item.price>OrderItem.price)) print [order.customer_name for order in result] diff --git a/examples/collections/large_collection.py b/examples/collections/large_collection.py index 3c53db121c..203aa6d230 100644 --- a/examples/collections/large_collection.py +++ b/examples/collections/large_collection.py @@ -1,6 +1,11 @@ -"""illlustrates techniques for dealing with very large collections""" +"""illlustrates techniques for dealing with very large collections. + +Also see the docs regarding the new "dynamic" relation option, which +presents a more refined version of some of these patterns. +""" from sqlalchemy import * +from sqlalchemy.orm import * meta = MetaData('sqlite://') meta.bind.echo = True @@ -60,7 +65,7 @@ sess.clear() # reload. load the org and some child members print "-------------------------\nload subset of members" org = sess.query(Organization).get(org.org_id) -members = org.member_query.filter_by(member_table.c.name.like('%member t%')).list() +members = org.member_query.filter(member_table.c.name.like('%member t%')).all() print members sess.clear() diff --git a/examples/pickle/custom_pickler.py b/examples/pickle/custom_pickler.py index 0a32bfd03a..1c88c88e82 100644 --- a/examples/pickle/custom_pickler.py +++ b/examples/pickle/custom_pickler.py @@ -11,7 +11,7 @@ meta = MetaData('sqlite://') meta.bind.echo = True class MyExt(MapperExtension): - def populate_instance(self, mapper, selectcontext, row, instance, identitykey, isnew): + def populate_instance(self, mapper, selectcontext, row, instance, **flags): MyPickler.sessions.current = selectcontext.session return EXT_CONTINUE def before_insert(self, mapper, connection, instance): diff --git a/lib/sqlalchemy/databases/firebird.py b/lib/sqlalchemy/databases/firebird.py index a4262d9ca8..d520046d06 100644 --- a/lib/sqlalchemy/databases/firebird.py +++ b/lib/sqlalchemy/databases/firebird.py @@ -308,7 +308,10 @@ class FBCompiler(compiler.DefaultCompiler): def uses_sequences_for_inserts(self): return True - + + def visit_sequence(self, seq): + return "gen_id(" + seq.name + ", 1)" + def get_select_precolumns(self, select): """Called when building a ``SELECT`` statement, position is just before column list Firebird puts the limit and offset right diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 6f3badb445..c7364721f4 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -392,11 +392,9 @@ class ExecutionContext(object): raise NotImplementedError() def lastrow_has_defaults(self): - """Return True if the last row INSERTED via a compiled insert statement contained PassiveDefaults. + """Return True if the last INSERT or UPDATE row contained + inlined or database-side defaults. - The presence of PassiveDefaults indicates that the database - inserted data beyond that which we passed to the query - programmatically. """ raise NotImplementedError() @@ -1349,7 +1347,14 @@ class ResultProxy(object): """ return self.context.lastrow_has_defaults() + + def postfetch_cols(self): + """Return ``postfetch_cols()`` from the underlying ExecutionContext. + See ExecutionContext for details. + """ + return self.context.postfetch_cols() + def supports_sane_rowcount(self): """Return ``supports_sane_rowcount`` from the dialect. diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index 76676e4e53..9ae83460da 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1190,7 +1190,7 @@ class Mapper(object): which will populate those attributes in one query when next accessed. """ - postfetch_cols = resultproxy.context.postfetch_cols().union(util.Set(value_params.keys())) + postfetch_cols = resultproxy.postfetch_cols().union(util.Set(value_params.keys())) deferred_props = [] for c in table.c: diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 617b2468a1..7f9d0e31b4 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -614,7 +614,7 @@ class DefaultCompiler(engine.Compiled, visitors.ClauseVisitor): return False def visit_sequence(self, seq): - raise NotImplementedError() + return None def visit_insert(self, insert_stmt): @@ -688,32 +688,26 @@ class DefaultCompiler(engine.Compiled, visitors.ClauseVisitor): values.append((c, value)) elif isinstance(c, schema.Column): if self.isinsert: - if isinstance(c.default, schema.ColumnDefault): - if self.inline and isinstance(c.default.arg, sql.ClauseElement): + if c.primary_key and self.uses_sequences_for_inserts() and not self.inline: + values.append((c, create_bind_param(c, None))) + self.prefetch.add(c) + elif isinstance(c.default, schema.ColumnDefault): + if isinstance(c.default.arg, sql.ClauseElement): values.append((c, self.process(c.default.arg))) self.postfetch.add(c) else: values.append((c, create_bind_param(c, None))) self.prefetch.add(c) elif isinstance(c.default, schema.PassiveDefault): - if c.primary_key and self.uses_sequences_for_inserts() and not self.inline: - values.append((c, create_bind_param(c, None))) - self.prefetch.add(c) - else: + self.postfetch.add(c) + elif isinstance(c.default, schema.Sequence): + proc = self.process(c.default) + if proc is not None: + values.append((c, proc)) self.postfetch.add(c) - elif (c.primary_key or isinstance(c.default, schema.Sequence)) and self.uses_sequences_for_inserts(): - if self.inline: - if c.default is not None: - proc = self.process(c.default) - if proc is not None: - values.append((c, proc)) - self.postfetch.add(c) - else: - values.append((c, create_bind_param(c, None))) - self.prefetch.add(c) elif self.isupdate: if isinstance(c.onupdate, schema.ColumnDefault): - if self.inline and isinstance(c.onupdate.arg, sql.ClauseElement): + if isinstance(c.onupdate.arg, sql.ClauseElement): values.append((c, self.process(c.onupdate.arg))) self.postfetch.add(c) else: diff --git a/test/dialect/postgres.py b/test/dialect/postgres.py index 3a1c978ac4..06cebaf17d 100644 --- a/test/dialect/postgres.py +++ b/test/dialect/postgres.py @@ -537,8 +537,7 @@ class TimezoneTest(AssertMixin): somedate = testbase.db.connect().scalar(func.current_timestamp().select()) tztable.insert().execute(id=1, name='row1', date=somedate) c = tztable.update(tztable.c.id==1).execute(name='newname') - x = c.last_updated_params() - print x['date'] == somedate + print tztable.select(tztable.c.id==1).execute().fetchone() @testing.supported('postgres') def test_without_timezone(self): @@ -546,8 +545,7 @@ class TimezoneTest(AssertMixin): somedate = datetime.datetime(2005, 10,20, 11, 52, 00) notztable.insert().execute(id=1, name='row1', date=somedate) c = notztable.update(notztable.c.id==1).execute(name='newname') - x = c.last_updated_params() - print x['date'] == somedate + print notztable.select(tztable.c.id==1).execute().fetchone() class ArrayTest(AssertMixin): @testing.supported('postgres') diff --git a/test/orm/unitofwork.py b/test/orm/unitofwork.py index dc5dfd1481..6dd4a08e83 100644 --- a/test/orm/unitofwork.py +++ b/test/orm/unitofwork.py @@ -602,7 +602,7 @@ class DefaultTest(ORMTest): default_table = Table('default_test', metadata, Column('id', Integer, Sequence("dt_seq", optional=True), primary_key=True), Column('hoho', hohotype, PassiveDefault(str(self.hohoval))), - Column('counter', Integer, PassiveDefault("7")), + Column('counter', Integer, default=func.length("1234567")), Column('foober', String(30), default="im foober", onupdate="im the update") ) diff --git a/test/sql/defaults.py b/test/sql/defaults.py index 1dbd60d57a..7f23466723 100644 --- a/test/sql/defaults.py +++ b/test/sql/defaults.py @@ -132,7 +132,7 @@ class DefaultTest(PersistTest): def testinsert(self): r = t.insert().execute() assert r.lastrow_has_defaults() - assert util.Set(r.context.postfetch_cols()) == util.Set([t.c.col5, t.c.col4]) + assert util.Set(r.context.postfetch_cols()) == util.Set([t.c.col3, t.c.col5, t.c.col4, t.c.col6]) r = t.insert(inline=True).execute() assert r.lastrow_has_defaults()