From: Mike Bayer Date: Thu, 11 Apr 2019 16:43:47 +0000 (-0400) Subject: initial 2.0 setup X-Git-Tag: rel_2_0_0b1~689^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f05d3ddba4c1edae00dd863af60e8fbb13eb7091;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git initial 2.0 setup Adapted from 55e64f857daeb6057b85ff67297a774b when we previously started a 2.0 branch. Change-Id: Ib5af75df94b23104eebe0e918adcf979d798ea3b --- diff --git a/doc/build/changelog/changelog_20.rst b/doc/build/changelog/changelog_20.rst new file mode 100644 index 0000000000..d72e5a4770 --- /dev/null +++ b/doc/build/changelog/changelog_20.rst @@ -0,0 +1,13 @@ +============= +2.0 Changelog +============= + +.. changelog_imports:: + + .. include:: changelog_14.rst + :start-line: 5 + + +.. changelog:: + :version: 2.0.0b1 + :include_notes_from: unreleased_14 diff --git a/doc/build/changelog/index.rst b/doc/build/changelog/index.rst index 1f5dec176c..101585f158 100644 --- a/doc/build/changelog/index.rst +++ b/doc/build/changelog/index.rst @@ -12,7 +12,7 @@ Current Migration Guide .. toctree:: :titlesonly: - migration_14 + migration_20 SQLAlchemy 2.0 Overview and Status ---------------------------------- @@ -28,6 +28,7 @@ Change logs .. toctree:: :titlesonly: + changelog_20 changelog_14 changelog_13 changelog_12 diff --git a/doc/build/changelog/unreleased_20/README.txt b/doc/build/changelog/unreleased_20/README.txt new file mode 100644 index 0000000000..1d2b3446e4 --- /dev/null +++ b/doc/build/changelog/unreleased_20/README.txt @@ -0,0 +1,12 @@ +Individual per-changelog files go here +in .rst format, which are pulled in by +changelog (version 0.4.0 or higher) to +be rendered into the changelog_xx.rst file. +At release time, the files here are removed and written +directly into the changelog. + +Rationale is so that multiple changes being merged +into gerrit don't produce conflicts. Note that +gerrit does not support custom merge handlers unlike +git itself. + diff --git a/doc/build/conf.py b/doc/build/conf.py index 169d695d0f..cf94c88524 100644 --- a/doc/build/conf.py +++ b/doc/build/conf.py @@ -201,11 +201,11 @@ copyright = u"2007-2021, the SQLAlchemy authors and contributors" # noqa # built documents. # # The short X.Y version. -version = "1.4" +version = "2.0" # The full version, including alpha/beta/rc tags. -release = "1.4.26" +release = "2.0.0b1" -release_date = "October 19, 2021" +release_date = None site_base = os.environ.get("RTD_SITE_BASE", "https://www.sqlalchemy.org") site_adapter_template = "docs_adapter.mako" diff --git a/doc/build/core/index.rst b/doc/build/core/index.rst index aaa63ca265..40180bef46 100644 --- a/doc/build/core/index.rst +++ b/doc/build/core/index.rst @@ -11,7 +11,6 @@ Language provides a schema-centric usage paradigm. .. toctree:: :maxdepth: 2 - tutorial expression_api schema types diff --git a/doc/build/core/tutorial.rst b/doc/build/core/tutorial.rst deleted file mode 100644 index 7a91e39a3f..0000000000 --- a/doc/build/core/tutorial.rst +++ /dev/null @@ -1,2614 +0,0 @@ -.. _sqlexpression_toplevel: - -========================================== -SQL Expression Language Tutorial (1.x API) -========================================== - -.. admonition:: About this document - - This tutorial covers the well known SQLAlchemy Core API - that has been in use for many years. As of SQLAlchemy 1.4, there are two - distinct styles of Core use known as :term:`1.x style` and :term:`2.0 - style`, the latter of which makes some adjustments mostly in the area - of how transactions are controlled as well as narrows down the patterns - for how SQL statement constructs are executed. - - The plan is that in SQLAlchemy 2.0, those elements of 1.x style - Core use will be removed, after a deprecation phase that continues - throughout the 1.4 series. For ORM use, some elements of 1.x style - will still be available; see the :ref:`migration_20_toplevel` document - for a complete overview. - - The tutorial here is applicable to users who want to learn how SQLAlchemy - Core has been used for many years, particularly those users working with - existing applications or related learning material that is in 1.x style. - - For an introduction to SQLAlchemy Core from the new 1.4/2.0 perspective, - see :ref:`unified_tutorial`. - - .. seealso:: - - :ref:`migration_20_toplevel` - - :ref:`unified_tutorial` - - -The SQLAlchemy Expression Language presents a system of representing -relational database structures and expressions using Python constructs. These -constructs are modeled to resemble those of the underlying database as closely -as possible, while providing a modicum of abstraction of the various -implementation differences between database backends. While the constructs -attempt to represent equivalent concepts between backends with consistent -structures, they do not conceal useful concepts that are unique to particular -subsets of backends. The Expression Language therefore presents a method of -writing backend-neutral SQL expressions, but does not attempt to enforce that -expressions are backend-neutral. - -The Expression Language is in contrast to the Object Relational Mapper, which -is a distinct API that builds on top of the Expression Language. Whereas the -ORM, introduced in :ref:`ormtutorial_toplevel`, presents a high level and -abstracted pattern of usage, which itself is an example of applied usage of -the Expression Language, the Expression Language presents a system of -representing the primitive constructs of the relational database directly -without opinion. - -While there is overlap among the usage patterns of the ORM and the Expression -Language, the similarities are more superficial than they may at first appear. -One approaches the structure and content of data from the perspective of a -user-defined `domain model -`_ which is transparently -persisted and refreshed from its underlying storage model. The other -approaches it from the perspective of literal schema and SQL expression -representations which are explicitly composed into messages consumed -individually by the database. - -A successful application may be constructed using the Expression Language -exclusively, though the application will need to define its own system of -translating application concepts into individual database messages and from -individual database result sets. Alternatively, an application constructed -with the ORM may, in advanced scenarios, make occasional usage of the -Expression Language directly in certain areas where specific database -interactions are required. - -The following tutorial is in doctest format, meaning each ``>>>`` line -represents something you can type at a Python command prompt, and the -following text represents the expected return value. The tutorial has no -prerequisites. - -Version Check -============= - - -A quick check to verify that we are on at least **version 1.4** of SQLAlchemy: - -.. sourcecode:: pycon+sql - - >>> import sqlalchemy - >>> sqlalchemy.__version__ # doctest: +SKIP - 1.4.0 - -Connecting -========== - -For this tutorial we will use an in-memory-only SQLite database. This is an -easy way to test things without needing to have an actual database defined -anywhere. To connect we use :func:`~sqlalchemy.create_engine`: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import create_engine - >>> engine = create_engine('sqlite:///:memory:', echo=True) - -The ``echo`` flag is a shortcut to setting up SQLAlchemy logging, which is -accomplished via Python's standard ``logging`` module. With it enabled, we'll -see all the generated SQL produced. If you are working through this tutorial -and want less output generated, set it to ``False``. This tutorial will format -the SQL behind a popup window so it doesn't get in our way; just click the -"SQL" links to see what's being generated. - -The return value of :func:`_sa.create_engine` is an instance of -:class:`_engine.Engine`, and it represents the core interface to the -database, adapted through a :term:`dialect` that handles the details -of the database and :term:`DBAPI` in use. In this case the SQLite -dialect will interpret instructions to the Python built-in ``sqlite3`` -module. - -.. sidebar:: Lazy Connecting - - The :class:`_engine.Engine`, when first returned by :func:`_sa.create_engine`, - has not actually tried to connect to the database yet; that happens - only the first time it is asked to perform a task against the database. - -The first time a method like :meth:`_engine.Engine.execute` or :meth:`_engine.Engine.connect` -is called, the :class:`_engine.Engine` establishes a real :term:`DBAPI` connection to the -database, which is then used to emit the SQL. - -.. seealso:: - - :ref:`database_urls` - includes examples of :func:`_sa.create_engine` - connecting to several kinds of databases with links to more information. - -Define and Create Tables -======================== - -The SQL Expression Language constructs its expressions in most cases against -table columns. In SQLAlchemy, a column is most often represented by an object -called :class:`~sqlalchemy.schema.Column`, and in all cases a -:class:`~sqlalchemy.schema.Column` is associated with a -:class:`~sqlalchemy.schema.Table`. A collection of -:class:`~sqlalchemy.schema.Table` objects and their associated child objects -is referred to as **database metadata**. In this tutorial we will explicitly -lay out several :class:`~sqlalchemy.schema.Table` objects, but note that SA -can also "import" whole sets of :class:`~sqlalchemy.schema.Table` objects -automatically from an existing database (this process is called **table -reflection**). - -We define our tables all within a catalog called -:class:`~sqlalchemy.schema.MetaData`, using the -:class:`~sqlalchemy.schema.Table` construct, which resembles regular SQL -CREATE TABLE statements. We'll make two tables, one of which represents -"users" in an application, and another which represents zero or more "email -addresses" for each row in the "users" table: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey - >>> metadata_obj = MetaData() - >>> users = Table('users', metadata_obj, - ... Column('id', Integer, primary_key=True), - ... Column('name', String), - ... Column('fullname', String), - ... ) - - >>> addresses = Table('addresses', metadata_obj, - ... Column('id', Integer, primary_key=True), - ... Column('user_id', None, ForeignKey('users.id')), - ... Column('email_address', String, nullable=False) - ... ) - -All about how to define :class:`~sqlalchemy.schema.Table` objects, as well as -how to create them from an existing database automatically, is described in -:ref:`metadata_toplevel`. - -Next, to tell the :class:`~sqlalchemy.schema.MetaData` we'd actually like to -create our selection of tables for real inside the SQLite database, we use -:func:`~sqlalchemy.schema.MetaData.create_all`, passing it the ``engine`` -instance which points to our database. This will check for the presence of -each table first before creating, so it's safe to call multiple times: - -.. sourcecode:: pycon+sql - - {sql}>>> metadata_obj.create_all(engine) - BEGIN... - CREATE TABLE users ( - id INTEGER NOT NULL, - name VARCHAR, - fullname VARCHAR, - PRIMARY KEY (id) - ) - [...] () - CREATE TABLE addresses ( - id INTEGER NOT NULL, - user_id INTEGER, - email_address VARCHAR NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY(user_id) REFERENCES users (id) - ) - [...] () - COMMIT - -.. note:: - - Users familiar with the syntax of CREATE TABLE may notice that the - VARCHAR columns were generated without a length; on SQLite and PostgreSQL, - this is a valid datatype, but on others, it's not allowed. So if running - this tutorial on one of those databases, and you wish to use SQLAlchemy to - issue CREATE TABLE, a "length" may be provided to the :class:`~sqlalchemy.types.String` type as - below:: - - Column('name', String(50)) - - The length field on :class:`~sqlalchemy.types.String`, as well as similar precision/scale fields - available on :class:`~sqlalchemy.types.Integer`, :class:`~sqlalchemy.types.Numeric`, etc. are not referenced by - SQLAlchemy other than when creating tables. - - Additionally, Firebird and Oracle require sequences to generate new - primary key identifiers, and SQLAlchemy doesn't generate or assume these - without being instructed. For that, you use the :class:`~sqlalchemy.schema.Sequence` construct:: - - from sqlalchemy import Sequence - Column('id', Integer, Sequence('user_id_seq'), primary_key=True) - - A full, foolproof :class:`~sqlalchemy.schema.Table` is therefore:: - - users = Table('users', metadata_obj, - Column('id', Integer, Sequence('user_id_seq'), primary_key=True), - Column('name', String(50)), - Column('fullname', String(50)), - Column('nickname', String(50)) - ) - - We include this more verbose :class:`_schema.Table` construct separately - to highlight the difference between a minimal construct geared primarily - towards in-Python usage only, versus one that will be used to emit CREATE - TABLE statements on a particular set of backends with more stringent - requirements. - -.. _coretutorial_insert_expressions: - -Insert Expressions -================== - -The first SQL expression we'll create is the -:class:`~sqlalchemy.sql.expression.Insert` construct, which represents an -INSERT statement. This is typically created relative to its target table:: - - >>> ins = users.insert() - -To see a sample of the SQL this construct produces, use the ``str()`` -function:: - - >>> str(ins) - 'INSERT INTO users (id, name, fullname) VALUES (:id, :name, :fullname)' - -Notice above that the INSERT statement names every column in the ``users`` -table. This can be limited by using the ``values()`` method, which establishes -the VALUES clause of the INSERT explicitly:: - - >>> ins = users.insert().values(name='jack', fullname='Jack Jones') - >>> str(ins) - 'INSERT INTO users (name, fullname) VALUES (:name, :fullname)' - -Above, while the ``values`` method limited the VALUES clause to just two -columns, the actual data we placed in ``values`` didn't get rendered into the -string; instead we got named bind parameters. As it turns out, our data *is* -stored within our :class:`~sqlalchemy.sql.expression.Insert` construct, but it -typically only comes out when the statement is actually executed; since the -data consists of literal values, SQLAlchemy automatically generates bind -parameters for them. We can peek at this data for now by looking at the -compiled form of the statement:: - - >>> ins.compile().params # doctest: +SKIP - {'fullname': 'Jack Jones', 'name': 'jack'} - -Executing -========= - -The interesting part of an :class:`~sqlalchemy.sql.expression.Insert` is -executing it. This is performed using a database connection, which is -represented by the :class:`_engine.Connection` object. To acquire a -connection, we will use the :meth:`_engine.Engine.connect` method:: - - >>> conn = engine.connect() - >>> conn - - -The :class:`~sqlalchemy.engine.Connection` object represents an actively -checked out DBAPI connection resource. Lets feed it our -:class:`~sqlalchemy.sql.expression.Insert` object and see what happens: - -.. sourcecode:: pycon+sql - - >>> result = conn.execute(ins) - {opensql}INSERT INTO users (name, fullname) VALUES (?, ?) - [...] ('jack', 'Jack Jones') - COMMIT - -So the INSERT statement was now issued to the database. Although we got -positional "qmark" bind parameters instead of "named" bind parameters in the -output. How come ? Because when executed, the -:class:`~sqlalchemy.engine.Connection` used the SQLite **dialect** to -help generate the statement; when we use the ``str()`` function, the statement -isn't aware of this dialect, and falls back onto a default which uses named -parameters. We can view this manually as follows: - -.. sourcecode:: pycon+sql - - >>> ins.bind = engine - >>> str(ins) - 'INSERT INTO users (name, fullname) VALUES (?, ?)' - -What about the ``result`` variable we got when we called ``execute()`` ? As -the SQLAlchemy :class:`~sqlalchemy.engine.Connection` object references a -DBAPI connection, the result, known as a -:class:`~sqlalchemy.engine.CursorResult` object, is analogous to the DBAPI -cursor object. In the case of an INSERT, we can get important information from -it, such as the primary key values which were generated from our statement -using :attr:`_engine.CursorResult.inserted_primary_key`: - -.. sourcecode:: pycon+sql - - >>> result.inserted_primary_key - (1,) - -The value of ``1`` was automatically generated by SQLite, but only because we -did not specify the ``id`` column in our -:class:`~sqlalchemy.sql.expression.Insert` statement; otherwise, our explicit -value would have been used. In either case, SQLAlchemy always knows how to get -at a newly generated primary key value, even though the method of generating -them is different across different databases; each database's -:class:`~sqlalchemy.engine.interfaces.Dialect` knows the specific steps needed to -determine the correct value (or values; note that -:attr:`_engine.CursorResult.inserted_primary_key` -returns a list so that it supports composite primary keys). Methods here -range from using ``cursor.lastrowid``, to selecting from a database-specific -function, to using ``INSERT..RETURNING`` syntax; this all occurs transparently. - -.. _execute_multiple: - -Executing Multiple Statements -============================= - -Our insert example above was intentionally a little drawn out to show some -various behaviors of expression language constructs. In the usual case, an -:class:`~sqlalchemy.sql.expression.Insert` statement is usually compiled -against the parameters sent to the ``execute()`` method on -:class:`~sqlalchemy.engine.Connection`, so that there's no need to use -the ``values`` keyword with :class:`~sqlalchemy.sql.expression.Insert`. Lets -create a generic :class:`~sqlalchemy.sql.expression.Insert` statement again -and use it in the "normal" way: - -.. sourcecode:: pycon+sql - - >>> ins = users.insert() - >>> conn.execute(ins, {"id": 2, "name":"wendy", "fullname": "Wendy Williams"}) - {opensql}INSERT INTO users (id, name, fullname) VALUES (?, ?, ?) - [...] (2, 'wendy', 'Wendy Williams') - COMMIT - {stop} - -Above, because we specified all three columns in the ``execute()`` method, -the compiled :class:`_expression.Insert` included all three -columns. The :class:`_expression.Insert` statement is compiled -at execution time based on the parameters we specified; if we specified fewer -parameters, the :class:`_expression.Insert` would have fewer -entries in its VALUES clause. - -To issue many inserts using DBAPI's ``executemany()`` method, we can send in a -list of dictionaries each containing a distinct set of parameters to be -inserted, as we do here to add some email addresses: - -.. sourcecode:: pycon+sql - - >>> conn.execute(addresses.insert(), [ - ... {'user_id': 1, 'email_address' : 'jack@yahoo.com'}, - ... {'user_id': 1, 'email_address' : 'jack@msn.com'}, - ... {'user_id': 2, 'email_address' : 'www@www.org'}, - ... {'user_id': 2, 'email_address' : 'wendy@aol.com'}, - ... ]) - {opensql}INSERT INTO addresses (user_id, email_address) VALUES (?, ?) - [...] ((1, 'jack@yahoo.com'), (1, 'jack@msn.com'), (2, 'www@www.org'), (2, 'wendy@aol.com')) - COMMIT - {stop} - -Above, we again relied upon SQLite's automatic generation of primary key -identifiers for each ``addresses`` row. - -When executing multiple sets of parameters, each dictionary must have the -**same** set of keys; i.e. you cant have fewer keys in some dictionaries than -others. This is because the :class:`~sqlalchemy.sql.expression.Insert` -statement is compiled against the **first** dictionary in the list, and it's -assumed that all subsequent argument dictionaries are compatible with that -statement. - -The "executemany" style of invocation is available for each of the -:func:`_expression.insert`, :func:`_expression.update` and :func:`_expression.delete` constructs. - - -.. _coretutorial_selecting: - -Selecting -========= - -We began with inserts just so that our test database had some data in it. The -more interesting part of the data is selecting it! We'll cover UPDATE and -DELETE statements later. The primary construct used to generate SELECT -statements is the :func:`_expression.select` function: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import select - >>> s = select(users) - >>> result = conn.execute(s) - {opensql}SELECT users.id, users.name, users.fullname - FROM users - [...] () - -Above, we issued a basic :func:`_expression.select` call, placing the ``users`` table -within the COLUMNS clause of the select, and then executing. SQLAlchemy -expanded the ``users`` table into the set of each of its columns, and also -generated a FROM clause for us. - -.. versionchanged:: 1.4 The :func:`_expression.select` construct now accepts - column arguments positionally, as ``select(*args)``. The previous style - of ``select()`` accepting a list of column elements is now deprecated. - See :ref:`change_5284`. - -The result returned is again a -:class:`~sqlalchemy.engine.CursorResult` object, which acts much like a -DBAPI cursor, including methods such as -:func:`~sqlalchemy.engine.CursorResult.fetchone` and -:func:`~sqlalchemy.engine.CursorResult.fetchall`. These methods return -row objects, which are provided via the :class:`.Row` class. The -result object can be iterated directly in order to provide an iterator -of :class:`.Row` objects: - -.. sourcecode:: pycon+sql - - >>> for row in result: - ... print(row) - (1, u'jack', u'Jack Jones') - (2, u'wendy', u'Wendy Williams') - -Above, we see that printing each :class:`.Row` produces a simple -tuple-like result. The most canonical way in Python to access the values -of these tuples as rows are fetched is through tuple assignment: - -.. sourcecode:: pycon+sql - - {sql}>>> result = conn.execute(s) - SELECT users.id, users.name, users.fullname - FROM users - [...] () - - {stop}>>> for id, name, fullname in result: - ... print("name:", name, "; fullname: ", fullname) - name: jack ; fullname: Jack Jones - name: wendy ; fullname: Wendy Williams - -The :class:`.Row` object actually behaves like a Python named tuple, so -we may also access these attributes from the row itself using attribute -access: - -.. sourcecode:: pycon+sql - - {sql}>>> result = conn.execute(s) - SELECT users.id, users.name, users.fullname - FROM users - [...] () - - {stop}>>> for row in result: - ... print("name:", row.name, "; fullname: ", row.fullname) - name: jack ; fullname: Jack Jones - name: wendy ; fullname: Wendy Williams - -To access columns via name using strings, either when the column name is -programmatically generated, or contains non-ascii characters, the -:attr:`.Row._mapping` view may be used that provides dictionary-like access: - -.. sourcecode:: pycon+sql - - {sql}>>> result = conn.execute(s) - SELECT users.id, users.name, users.fullname - FROM users - [...] () - - {stop}>>> row = result.fetchone() - >>> print("name:", row._mapping['name'], "; fullname:", row._mapping['fullname']) - name: jack ; fullname: Jack Jones - -.. deprecated:: 1.4 - - In versions of SQLAlchemy prior to 1.4, the above access using - :attr:`.Row._mapping` would proceed against the row object itself, that - is:: - - row = result.fetchone() - name, fullname = row["name"], row["fullname"] - - This pattern is now deprecated and will be removed in SQLAlchemy 2.0, so - that the :class:`.Row` object may now behave fully like a Python named - tuple. - -.. versionchanged:: 1.4 Added :attr:`.Row._mapping` which provides for - dictionary-like access to a :class:`.Row`, superseding the use of string/ - column keys against the :class:`.Row` object directly. - -As the :class:`.Row` is a tuple, sequence (i.e. integer or slice) access -may be used as well: - -.. sourcecode:: pycon+sql - - >>> row = result.fetchone() - >>> print("name:", row[1], "; fullname:", row[2]) - name: wendy ; fullname: Wendy Williams - -A more specialized method of column access is to use the SQL construct that -directly corresponds to a particular column as the mapping key; in this -example, it means we would use the :class:`_schema.Column` objects selected in our -SELECT directly as keys in conjunction with the :attr:`.Row._mapping` -collection: - -.. sourcecode:: pycon+sql - - {sql}>>> for row in conn.execute(s): - ... print("name:", row._mapping[users.c.name], "; fullname:", row._mapping[users.c.fullname]) - SELECT users.id, users.name, users.fullname - FROM users - [...] () - {stop}name: jack ; fullname: Jack Jones - name: wendy ; fullname: Wendy Williams - -.. sidebar:: Results and Rows are changing - - The :class:`.Row` class was known as ``RowProxy`` and the - :class:`_engine.CursorResult` class was known as ``ResultProxy``, for all - SQLAlchemy versions through 1.3. In 1.4, the objects returned by - :class:`_engine.CursorResult` are actually a subclass of :class:`.Row` known as - :class:`.LegacyRow`. See :ref:`change_4710_core` for background on this - change. - -The :class:`_engine.CursorResult` object features "auto-close" behavior that closes the -underlying DBAPI ``cursor`` object when all pending result rows have been -fetched. If a :class:`_engine.CursorResult` is to be discarded before such an -autoclose has occurred, it can be explicitly closed using the -:meth:`_engine.CursorResult.close` method: - -.. sourcecode:: pycon+sql - - >>> result.close() - -Selecting Specific Columns -=========================== - -If we'd like to more carefully control the columns which are placed in the -COLUMNS clause of the select, we reference individual -:class:`~sqlalchemy.schema.Column` objects from our -:class:`~sqlalchemy.schema.Table`. These are available as named attributes off -the ``c`` attribute of the :class:`~sqlalchemy.schema.Table` object: - -.. sourcecode:: pycon+sql - - >>> s = select(users.c.name, users.c.fullname) - {sql}>>> result = conn.execute(s) - SELECT users.name, users.fullname - FROM users - [...] () - {stop}>>> for row in result: - ... print(row) - (u'jack', u'Jack Jones') - (u'wendy', u'Wendy Williams') - -Lets observe something interesting about the FROM clause. Whereas the -generated statement contains two distinct sections, a "SELECT columns" part -and a "FROM table" part, our :func:`_expression.select` construct only has a list -containing columns. How does this work ? Let's try putting *two* tables into -our :func:`_expression.select` statement: - -.. sourcecode:: pycon+sql - - {sql}>>> for row in conn.execute(select(users, addresses)): - ... print(row) - SELECT users.id, users.name, users.fullname, addresses.id AS id_1, addresses.user_id, addresses.email_address - FROM users, addresses - [...] () - {stop}(1, u'jack', u'Jack Jones', 1, 1, u'jack@yahoo.com') - (1, u'jack', u'Jack Jones', 2, 1, u'jack@msn.com') - (1, u'jack', u'Jack Jones', 3, 2, u'www@www.org') - (1, u'jack', u'Jack Jones', 4, 2, u'wendy@aol.com') - (2, u'wendy', u'Wendy Williams', 1, 1, u'jack@yahoo.com') - (2, u'wendy', u'Wendy Williams', 2, 1, u'jack@msn.com') - (2, u'wendy', u'Wendy Williams', 3, 2, u'www@www.org') - (2, u'wendy', u'Wendy Williams', 4, 2, u'wendy@aol.com') - -It placed **both** tables into the FROM clause. But also, it made a real mess. -Those who are familiar with SQL joins know that this is a **Cartesian -product**; each row from the ``users`` table is produced against each row from -the ``addresses`` table. So to put some sanity into this statement, we need a -WHERE clause. We do that using :meth:`_expression.Select.where`: - -.. sourcecode:: pycon+sql - - >>> s = select(users, addresses).where(users.c.id == addresses.c.user_id) - {sql}>>> for row in conn.execute(s): - ... print(row) - SELECT users.id, users.name, users.fullname, addresses.id AS id_1, - addresses.user_id, addresses.email_address - FROM users, addresses - WHERE users.id = addresses.user_id - [...] () - {stop}(1, u'jack', u'Jack Jones', 1, 1, u'jack@yahoo.com') - (1, u'jack', u'Jack Jones', 2, 1, u'jack@msn.com') - (2, u'wendy', u'Wendy Williams', 3, 2, u'www@www.org') - (2, u'wendy', u'Wendy Williams', 4, 2, u'wendy@aol.com') - -So that looks a lot better, we added an expression to our :func:`_expression.select` -which had the effect of adding ``WHERE users.id = addresses.user_id`` to our -statement, and our results were managed down so that the join of ``users`` and -``addresses`` rows made sense. But let's look at that expression? It's using -just a Python equality operator between two different -:class:`~sqlalchemy.schema.Column` objects. It should be clear that something -is up. Saying ``1 == 1`` produces ``True``, and ``1 == 2`` produces ``False``, not -a WHERE clause. So lets see exactly what that expression is doing: - -.. sourcecode:: pycon+sql - - >>> users.c.id == addresses.c.user_id - - -Wow, surprise ! This is neither a ``True`` nor a ``False``. Well what is it ? - -.. sourcecode:: pycon+sql - - >>> str(users.c.id == addresses.c.user_id) - 'users.id = addresses.user_id' - -As you can see, the ``==`` operator is producing an object that is very much -like the :class:`_expression.Insert` and :func:`_expression.select` -objects we've made so far, thanks to Python's ``__eq__()`` builtin; you call -``str()`` on it and it produces SQL. By now, one can see that everything we -are working with is ultimately the same type of object. SQLAlchemy terms the -base class of all of these expressions as :class:`_expression.ColumnElement`. - -Operators -========= - -Since we've stumbled upon SQLAlchemy's operator paradigm, let's go through -some of its capabilities. We've seen how to equate two columns to each other: - -.. sourcecode:: pycon+sql - - >>> print(users.c.id == addresses.c.user_id) - users.id = addresses.user_id - -If we use a literal value (a literal meaning, not a SQLAlchemy clause object), -we get a bind parameter: - -.. sourcecode:: pycon+sql - - >>> print(users.c.id == 7) - users.id = :id_1 - -The ``7`` literal is embedded the resulting -:class:`_expression.ColumnElement`; we can use the same trick -we did with the :class:`~sqlalchemy.sql.expression.Insert` object to see it: - -.. sourcecode:: pycon+sql - - >>> (users.c.id == 7).compile().params - {u'id_1': 7} - -Most Python operators, as it turns out, produce a SQL expression here, like -equals, not equals, etc.: - -.. sourcecode:: pycon+sql - - >>> print(users.c.id != 7) - users.id != :id_1 - - >>> # None converts to IS NULL - >>> print(users.c.name == None) - users.name IS NULL - - >>> # reverse works too - >>> print('fred' > users.c.name) - users.name < :name_1 - -If we add two integer columns together, we get an addition expression: - -.. sourcecode:: pycon+sql - - >>> print(users.c.id + addresses.c.id) - users.id + addresses.id - -Interestingly, the type of the :class:`~sqlalchemy.schema.Column` is important! -If we use ``+`` with two string based columns (recall we put types like -:class:`~sqlalchemy.types.Integer` and :class:`~sqlalchemy.types.String` on -our :class:`~sqlalchemy.schema.Column` objects at the beginning), we get -something different: - -.. sourcecode:: pycon+sql - - >>> print(users.c.name + users.c.fullname) - users.name || users.fullname - -Where ``||`` is the string concatenation operator used on most databases. But -not all of them. MySQL users, fear not: - -.. sourcecode:: pycon+sql - - >>> print((users.c.name + users.c.fullname). - ... compile(bind=create_engine('mysql://'))) # doctest: +SKIP - concat(users.name, users.fullname) - -The above illustrates the SQL that's generated for an -:class:`~sqlalchemy.engine.Engine` that's connected to a MySQL database; -the ``||`` operator now compiles as MySQL's ``concat()`` function. - -If you have come across an operator which really isn't available, you can -always use the :meth:`.Operators.op` method; this generates whatever operator you need: - -.. sourcecode:: pycon+sql - - >>> print(users.c.name.op('tiddlywinks')('foo')) - users.name tiddlywinks :name_1 - -This function can also be used to make bitwise operators explicit. For example:: - - somecolumn.op('&')(0xff) - -is a bitwise AND of the value in ``somecolumn``. - -When using :meth:`.Operators.op`, the return type of the expression may be important, -especially when the operator is used in an expression that will be sent as a result -column. For this case, be sure to make the type explicit, if not what's -normally expected, using :func:`.type_coerce`:: - - from sqlalchemy import type_coerce - expr = type_coerce(somecolumn.op('-%>')('foo'), MySpecialType()) - stmt = select(expr) - - -For boolean operators, use the :meth:`.Operators.bool_op` method, which -will ensure that the return type of the expression is handled as boolean:: - - somecolumn.bool_op('-->')('some value') - - -Commonly Used Operators -------------------------- - - -Here's a rundown of some of the most common operators used in both the -Core expression language as well as in the ORM. Here we see expressions -that are most commonly present when using the :meth:`_sql.Select.where` method, -but can be used in other scenarios as well. - -A listing of all the column-level operations common to all column-like -objects is at :class:`.ColumnOperators`. - - -* :meth:`equals <.ColumnOperators.__eq__>`:: - - statement.where(users.c.name == 'ed') - -* :meth:`not equals <.ColumnOperators.__ne__>`:: - - statement.where(users.c.name != 'ed') - -* :meth:`LIKE <.ColumnOperators.like>`:: - - statement.where(users.c.name.like('%ed%')) - - .. note:: :meth:`.ColumnOperators.like` renders the LIKE operator, which - is case insensitive on some backends, and case sensitive - on others. For guaranteed case-insensitive comparisons, use - :meth:`.ColumnOperators.ilike`. - -* :meth:`ILIKE <.ColumnOperators.ilike>` (case-insensitive LIKE):: - - statement.where(users.c.name.ilike('%ed%')) - - .. note:: most backends don't support ILIKE directly. For those, - the :meth:`.ColumnOperators.ilike` operator renders an expression - combining LIKE with the LOWER SQL function applied to each operand. - -* :meth:`IN <.ColumnOperators.in_>`:: - - statement.where(users.c.name.in_(['ed', 'wendy', 'jack'])) - - # works with Select objects too: - statement.where.filter(users.c.name.in_( - select(users.c.name).where(users.c.name.like('%ed%')) - )) - - # use tuple_() for composite (multi-column) queries - from sqlalchemy import tuple_ - statement.where( - tuple_(users.c.name, users.c.nickname).\ - in_([('ed', 'edsnickname'), ('wendy', 'windy')]) - ) - -* :meth:`NOT IN <.ColumnOperators.not_in>`:: - - statement.where(~users.c.name.in_(['ed', 'wendy', 'jack'])) - -* :meth:`IS NULL <.ColumnOperators.is_>`:: - - statement.where(users.c. == None) - - # alternatively, if pep8/linters are a concern - statement.where(users.c.name.is_(None)) - -* :meth:`IS NOT NULL <.ColumnOperators.is_not>`:: - - statement.where(users.c.name != None) - - # alternatively, if pep8/linters are a concern - statement.where(users.c.name.is_not(None)) - -* :func:`AND <.sql.expression.and_>`:: - - # use and_() - from sqlalchemy import and_ - statement.where(and_(users.c.name == 'ed', users.c.fullname == 'Ed Jones')) - - # or send multiple expressions to .where() - statement.where(users.c.name == 'ed', users.c.fullname == 'Ed Jones') - - # or chain multiple where() calls - statement.where(users.c.name == 'ed').where(users.c.fullname == 'Ed Jones') - - .. note:: Make sure you use :func:`.and_` and **not** the - Python ``and`` operator! - -* :func:`OR <.sql.expression.or_>`:: - - from sqlalchemy import or_ - statement.where(or_(users.c.name == 'ed', users.c.name == 'wendy')) - - .. note:: Make sure you use :func:`.or_` and **not** the - Python ``or`` operator! - -* :meth:`MATCH <.ColumnOperators.match>`:: - - statement.where(users.c.name.match('wendy')) - - .. note:: - - :meth:`~.ColumnOperators.match` uses a database-specific ``MATCH`` - or ``CONTAINS`` function; its behavior will vary by backend and is not - available on some backends such as SQLite. - - -Operator Customization ----------------------- - -While :meth:`.Operators.op` is handy to get at a custom operator in a hurry, -the Core supports fundamental customization and extension of the operator system at -the type level. The behavior of existing operators can be modified on a per-type -basis, and new operations can be defined which become available for all column -expressions that are part of that particular type. See the section :ref:`types_operators` -for a description. - - - -Conjunctions -============ - - -We'd like to show off some of our operators inside of :func:`_expression.select` -constructs. But we need to lump them together a little more, so let's first -introduce some conjunctions. Conjunctions are those little words like AND and -OR that put things together. We'll also hit upon NOT. :func:`.and_`, :func:`.or_`, -and :func:`.not_` can work -from the corresponding functions SQLAlchemy provides (notice we also throw in -a :meth:`~.ColumnOperators.like`): - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import and_, or_, not_ - >>> print(and_( - ... users.c.name.like('j%'), - ... users.c.id == addresses.c.user_id, - ... or_( - ... addresses.c.email_address == 'wendy@aol.com', - ... addresses.c.email_address == 'jack@yahoo.com' - ... ), - ... not_(users.c.id > 5) - ... ) - ... ) - users.name LIKE :name_1 AND users.id = addresses.user_id AND - (addresses.email_address = :email_address_1 - OR addresses.email_address = :email_address_2) - AND users.id <= :id_1 - -And you can also use the re-jiggered bitwise AND, OR and NOT operators, -although because of Python operator precedence you have to watch your -parenthesis: - -.. sourcecode:: pycon+sql - - >>> print(users.c.name.like('j%') & (users.c.id == addresses.c.user_id) & - ... ( - ... (addresses.c.email_address == 'wendy@aol.com') | \ - ... (addresses.c.email_address == 'jack@yahoo.com') - ... ) \ - ... & ~(users.c.id>5) - ... ) - users.name LIKE :name_1 AND users.id = addresses.user_id AND - (addresses.email_address = :email_address_1 - OR addresses.email_address = :email_address_2) - AND users.id <= :id_1 - -So with all of this vocabulary, let's select all users who have an email -address at AOL or MSN, whose name starts with a letter between "m" and "z", -and we'll also generate a column containing their full name combined with -their email address. We will add two new constructs to this statement, -:meth:`~.ColumnOperators.between` and :meth:`_expression.ColumnElement.label`. -:meth:`~.ColumnOperators.between` produces a BETWEEN clause, and -:meth:`_expression.ColumnElement.label` is used in a column expression to produce labels using the ``AS`` -keyword; it's recommended when selecting from expressions that otherwise would -not have a name: - -.. sourcecode:: pycon+sql - - >>> s = select((users.c.fullname + - ... ", " + addresses.c.email_address). - ... label('title')).\ - ... where( - ... and_( - ... users.c.id == addresses.c.user_id, - ... users.c.name.between('m', 'z'), - ... or_( - ... addresses.c.email_address.like('%@aol.com'), - ... addresses.c.email_address.like('%@msn.com') - ... ) - ... ) - ... ) - >>> conn.execute(s).fetchall() - {opensql}SELECT users.fullname || ? || addresses.email_address AS title - FROM users, addresses - WHERE users.id = addresses.user_id AND users.name BETWEEN ? AND ? AND - (addresses.email_address LIKE ? OR addresses.email_address LIKE ?) - [...] (', ', 'm', 'z', '%@aol.com', '%@msn.com') - {stop}[(u'Wendy Williams, wendy@aol.com',)] - -Once again, SQLAlchemy figured out the FROM clause for our statement. In fact -it will determine the FROM clause based on all of its other bits; the columns -clause, the where clause, and also some other elements which we haven't -covered yet, which include ORDER BY, GROUP BY, and HAVING. - -A shortcut to using :func:`.and_` is to chain together multiple -:meth:`_expression.Select.where` clauses. The above can also be written as: - -.. sourcecode:: pycon+sql - - >>> s = select((users.c.fullname + - ... ", " + addresses.c.email_address). - ... label('title')).\ - ... where(users.c.id == addresses.c.user_id).\ - ... where(users.c.name.between('m', 'z')).\ - ... where( - ... or_( - ... addresses.c.email_address.like('%@aol.com'), - ... addresses.c.email_address.like('%@msn.com') - ... ) - ... ) - >>> conn.execute(s).fetchall() - {opensql}SELECT users.fullname || ? || addresses.email_address AS title - FROM users, addresses - WHERE users.id = addresses.user_id AND users.name BETWEEN ? AND ? AND - (addresses.email_address LIKE ? OR addresses.email_address LIKE ?) - [...] (', ', 'm', 'z', '%@aol.com', '%@msn.com') - {stop}[(u'Wendy Williams, wendy@aol.com',)] - -The way that we can build up a :func:`_expression.select` construct through successive -method calls is called :term:`method chaining`. - -.. _sqlexpression_text: - -Using Textual SQL -================= - -Our last example really became a handful to type. Going from what one -understands to be a textual SQL expression into a Python construct which -groups components together in a programmatic style can be hard. That's why -SQLAlchemy lets you just use strings, for those cases when the SQL -is already known and there isn't a strong need for the statement to support -dynamic features. The :func:`_expression.text` construct is used -to compose a textual statement that is passed to the database mostly -unchanged. Below, we create a :func:`_expression.text` object and execute it: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import text - >>> s = text( - ... "SELECT users.fullname || ', ' || addresses.email_address AS title " - ... "FROM users, addresses " - ... "WHERE users.id = addresses.user_id " - ... "AND users.name BETWEEN :x AND :y " - ... "AND (addresses.email_address LIKE :e1 " - ... "OR addresses.email_address LIKE :e2)") - >>> conn.execute(s, {"x":"m", "y":"z", "e1":"%@aol.com", "e2":"%@msn.com"}).fetchall() - {opensql}SELECT users.fullname || ', ' || addresses.email_address AS title - FROM users, addresses - WHERE users.id = addresses.user_id AND users.name BETWEEN ? AND ? AND - (addresses.email_address LIKE ? OR addresses.email_address LIKE ?) - [...] ('m', 'z', '%@aol.com', '%@msn.com') - {stop}[(u'Wendy Williams, wendy@aol.com',)] - -Above, we can see that bound parameters are specified in -:func:`_expression.text` using the named colon format; this format is -consistent regardless of database backend. To send values in for the -parameters, we passed them into the :meth:`_engine.Connection.execute` method -as additional arguments. - -Specifying Bound Parameter Behaviors ------------------------------------- - -The :func:`_expression.text` construct supports pre-established bound values -using the :meth:`_expression.TextClause.bindparams` method:: - - stmt = text("SELECT * FROM users WHERE users.name BETWEEN :x AND :y") - stmt = stmt.bindparams(x="m", y="z") - -The parameters can also be explicitly typed:: - - stmt = stmt.bindparams(bindparam("x", type_=String), bindparam("y", type_=String)) - result = conn.execute(stmt, {"x": "m", "y": "z"}) - -Typing for bound parameters is necessary when the type requires Python-side -or special SQL-side processing provided by the datatype. - -.. seealso:: - - :meth:`_expression.TextClause.bindparams` - full method description - -.. _sqlexpression_text_columns: - -Specifying Result-Column Behaviors ----------------------------------- - -We may also specify information about the result columns using the -:meth:`_expression.TextClause.columns` method; this method can be used to specify -the return types, based on name:: - - stmt = stmt.columns(id=Integer, name=String) - -or it can be passed full column expressions positionally, either typed -or untyped. In this case it's a good idea to list out the columns -explicitly within our textual SQL, since the correlation of our column -expressions to the SQL will be done positionally:: - - stmt = text("SELECT id, name FROM users") - stmt = stmt.columns(users.c.id, users.c.name) - -When we call the :meth:`_expression.TextClause.columns` method, we get back a -:class:`.TextAsFrom` object that supports the full suite of -:attr:`.TextAsFrom.c` and other "selectable" operations:: - - j = stmt.join(addresses, stmt.c.id == addresses.c.user_id) - - new_stmt = select(stmt.c.id, addresses.c.id).\ - select_from(j).where(stmt.c.name == 'x') - -The positional form of :meth:`_expression.TextClause.columns` is particularly useful -when relating textual SQL to existing Core or ORM models, because we can use -column expressions directly without worrying about name conflicts or other issues with the -result column names in the textual SQL: - -.. sourcecode:: pycon+sql - - >>> stmt = text("SELECT users.id, addresses.id, users.id, " - ... "users.name, addresses.email_address AS email " - ... "FROM users JOIN addresses ON users.id=addresses.user_id " - ... "WHERE users.id = 1").columns( - ... users.c.id, - ... addresses.c.id, - ... addresses.c.user_id, - ... users.c.name, - ... addresses.c.email_address - ... ) - >>> result = conn.execute(stmt) - {opensql}SELECT users.id, addresses.id, users.id, users.name, - addresses.email_address AS email - FROM users JOIN addresses ON users.id=addresses.user_id WHERE users.id = 1 - [...] () - {stop} - -Above, there's three columns in the result that are named "id", but since -we've associated these with column expressions positionally, the names aren't an issue -when the result-columns are fetched using the actual column object as a key. -Fetching the ``email_address`` column would be:: - - >>> row = result.fetchone() - >>> row._mapping[addresses.c.email_address] - 'jack@yahoo.com' - -If on the other hand we used a string column key, the usual rules of -name-based matching still apply, and we'd get an ambiguous column error for -the ``id`` value:: - - >>> row._mapping["id"] - Traceback (most recent call last): - ... - InvalidRequestError: Ambiguous column name 'id' in result set column descriptions - -It's important to note that while accessing columns from a result set using -:class:`_schema.Column` objects may seem unusual, it is in fact the only system -used by the ORM, which occurs transparently beneath the facade of the -:class:`~.orm.query.Query` object; in this way, the :meth:`_expression.TextClause.columns` method -is typically very applicable to textual statements to be used in an ORM -context. The example at :ref:`orm_tutorial_literal_sql` illustrates -a simple usage. - -.. versionadded:: 1.1 - - The :meth:`_expression.TextClause.columns` method now accepts column expressions - which will be matched positionally to a plain text SQL result set, - eliminating the need for column names to match or even be unique in the - SQL statement when matching table metadata or ORM models to textual SQL. - -.. seealso:: - - :meth:`_expression.TextClause.columns` - full method description - - :ref:`orm_tutorial_literal_sql` - integrating ORM-level queries with - :func:`_expression.text` - - -Using text() fragments inside bigger statements ------------------------------------------------ - -:func:`_expression.text` can also be used to produce fragments of SQL -that can be freely within a -:func:`_expression.select` object, which accepts :func:`_expression.text` -objects as an argument for most of its builder functions. -Below, we combine the usage of :func:`_expression.text` within a -:func:`_expression.select` object. The :func:`_expression.select` construct provides the "geometry" -of the statement, and the :func:`_expression.text` construct provides the -textual content within this form. We can build a statement without the -need to refer to any pre-established :class:`_schema.Table` metadata: - -.. sourcecode:: pycon+sql - - >>> s = select( - ... text("users.fullname || ', ' || addresses.email_address AS title") - ... ).\ - ... where( - ... and_( - ... text("users.id = addresses.user_id"), - ... text("users.name BETWEEN 'm' AND 'z'"), - ... text( - ... "(addresses.email_address LIKE :x " - ... "OR addresses.email_address LIKE :y)") - ... ) - ... ).select_from(text('users, addresses')) - >>> conn.execute(s, {"x": "%@aol.com", "y": "%@msn.com"}).fetchall() - {opensql}SELECT users.fullname || ', ' || addresses.email_address AS title - FROM users, addresses - WHERE users.id = addresses.user_id AND users.name BETWEEN 'm' AND 'z' - AND (addresses.email_address LIKE ? OR addresses.email_address LIKE ?) - [...] ('%@aol.com', '%@msn.com') - {stop}[(u'Wendy Williams, wendy@aol.com',)] - - -While :func:`_expression.text` can be used in the column list of a -:func:`_expression.select` object, it has some restriction when composing the -generated select, since it will not be in -:attr:`_expression.SelectBase.selected_columns` collection and will be omitted -from the ``.c`` collection of subqueries. The next section will introduce the -:func:`_expression.literal_column` construct which is the better choice to -express individual column names as SQL fragments. - - - -.. _sqlexpression_literal_column: - -Using More Specific Text with :func:`.table`, :func:`_expression.literal_column`, and :func:`_expression.column` ------------------------------------------------------------------------------------------------------------------ -We can move our level of structure back in the other direction too, -by using :func:`_expression.column`, :func:`_expression.literal_column`, -and :func:`_expression.table` for some of the -key elements of our statement. Using these constructs, we can get -some more expression capabilities than if we used :func:`_expression.text` -directly, as they provide to the Core more information about how the strings -they store are to be used, but still without the need to get into full -:class:`_schema.Table` based metadata. Below, we also specify the :class:`.String` -datatype for two of the key :func:`_expression.literal_column` objects, -so that the string-specific concatenation operator becomes available. -We also use :func:`_expression.literal_column` in order to use table-qualified -expressions, e.g. ``users.fullname``, that will be rendered as is; -using :func:`_expression.column` implies an individual column name that may -be quoted: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import select, and_, text, String - >>> from sqlalchemy.sql import table, literal_column - >>> s = select( - ... literal_column("users.fullname", String) + - ... ', ' + - ... literal_column("addresses.email_address").label("title") - ... ).\ - ... where( - ... and_( - ... literal_column("users.id") == literal_column("addresses.user_id"), - ... text("users.name BETWEEN 'm' AND 'z'"), - ... text( - ... "(addresses.email_address LIKE :x OR " - ... "addresses.email_address LIKE :y)") - ... ) - ... ).select_from(table('users')).select_from(table('addresses')) - - >>> conn.execute(s, {"x":"%@aol.com", "y":"%@msn.com"}).fetchall() - {opensql}SELECT users.fullname || ? || addresses.email_address AS anon_1 - FROM users, addresses - WHERE users.id = addresses.user_id - AND users.name BETWEEN 'm' AND 'z' - AND (addresses.email_address LIKE ? OR addresses.email_address LIKE ?) - [...] (', ', '%@aol.com', '%@msn.com') - {stop}[(u'Wendy Williams, wendy@aol.com',)] - -.. _sqlexpression_order_by_label: - -Ordering or Grouping by a Label -------------------------------- - -One place where we sometimes want to use a string as a shortcut is when -our statement has some labeled column element that we want to refer to in -a place such as the "ORDER BY" or "GROUP BY" clause; other candidates include -fields within an "OVER" or "DISTINCT" clause. If we have such a label -in our :func:`_expression.select` construct, we can refer to it directly by passing the -string straight into :meth:`_expression.select.order_by` or :meth:`_expression.select.group_by`, -among others. This will refer to the named label and also prevent the -expression from being rendered twice. Label names that resolve to columns -are rendered fully: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import func - >>> stmt = select( - ... addresses.c.user_id, - ... func.count(addresses.c.id).label('num_addresses')).\ - ... group_by("user_id").order_by("user_id", "num_addresses") - - {sql}>>> conn.execute(stmt).fetchall() - SELECT addresses.user_id, count(addresses.id) AS num_addresses - FROM addresses GROUP BY addresses.user_id ORDER BY addresses.user_id, num_addresses - [...] () - {stop}[(1, 2), (2, 2)] - -We can use modifiers like :func:`.asc` or :func:`.desc` by passing the string -name: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import func, desc - >>> stmt = select( - ... addresses.c.user_id, - ... func.count(addresses.c.id).label('num_addresses')).\ - ... group_by("user_id").order_by("user_id", desc("num_addresses")) - - {sql}>>> conn.execute(stmt).fetchall() - SELECT addresses.user_id, count(addresses.id) AS num_addresses - FROM addresses GROUP BY addresses.user_id ORDER BY addresses.user_id, num_addresses DESC - [...] () - {stop}[(1, 2), (2, 2)] - -Note that the string feature here is very much tailored to when we have -already used the :meth:`_expression.ColumnElement.label` method to create a -specifically-named label. In other cases, we always want to refer to the -:class:`_expression.ColumnElement` object directly so that the expression system can -make the most effective choices for rendering. Below, we illustrate how using -the :class:`_expression.ColumnElement` eliminates ambiguity when we want to order -by a column name that appears more than once: - -.. sourcecode:: pycon+sql - - >>> u1a, u1b = users.alias(), users.alias() - >>> stmt = select(u1a, u1b).\ - ... where(u1a.c.name > u1b.c.name).\ - ... order_by(u1a.c.name) # using "name" here would be ambiguous - - {sql}>>> conn.execute(stmt).fetchall() - SELECT users_1.id, users_1.name, users_1.fullname, users_2.id AS id_1, - users_2.name AS name_1, users_2.fullname AS fullname_1 - FROM users AS users_1, users AS users_2 - WHERE users_1.name > users_2.name ORDER BY users_1.name - [...] () - {stop}[(2, u'wendy', u'Wendy Williams', 1, u'jack', u'Jack Jones')] - - - -.. _core_tutorial_aliases: - -Using Aliases and Subqueries -============================ - -The alias in SQL corresponds to a "renamed" version of a table or SELECT -statement, which occurs anytime you say "SELECT .. FROM sometable AS -someothername". The ``AS`` creates a new name for the table. Aliases are a key -construct as they allow any table or subquery to be referenced by a unique -name. In the case of a table, this allows the same table to be named in the -FROM clause multiple times. In the case of a SELECT statement, it provides a -parent name for the columns represented by the statement, allowing them to be -referenced relative to this name. - -In SQLAlchemy, any :class:`_schema.Table` or other :class:`_expression.FromClause` based -selectable can be turned into an alias using :meth:`_expression.FromClause.alias` method, -which produces an :class:`_expression.Alias` construct. :class:`_expression.Alias` is a -:class:`_expression.FromClause` object that refers to a mapping of :class:`_schema.Column` -objects via its :attr:`_expression.FromClause.c` collection, and can be used within the -FROM clause of any subsequent SELECT statement, by referring to its column -elements in the columns or WHERE clause of the statement, or through explicit -placement in the FROM clause, either directly or within a join. - -As an example, suppose we know that our user ``jack`` has two particular email -addresses. How can we locate jack based on the combination of those two -addresses? To accomplish this, we'd use a join to the ``addresses`` table, -once for each address. We create two :class:`_expression.Alias` constructs against -``addresses``, and then use them both within a :func:`_expression.select` construct: - -.. sourcecode:: pycon+sql - - >>> a1 = addresses.alias() - >>> a2 = addresses.alias() - >>> s = select(users).\ - ... where(and_( - ... users.c.id == a1.c.user_id, - ... users.c.id == a2.c.user_id, - ... a1.c.email_address == 'jack@msn.com', - ... a2.c.email_address == 'jack@yahoo.com' - ... )) - >>> conn.execute(s).fetchall() - {opensql}SELECT users.id, users.name, users.fullname - FROM users, addresses AS addresses_1, addresses AS addresses_2 - WHERE users.id = addresses_1.user_id - AND users.id = addresses_2.user_id - AND addresses_1.email_address = ? - AND addresses_2.email_address = ? - [...] ('jack@msn.com', 'jack@yahoo.com') - {stop}[(1, u'jack', u'Jack Jones')] - -Note that the :class:`_expression.Alias` construct generated the names ``addresses_1`` and -``addresses_2`` in the final SQL result. The generation of these names is determined -by the position of the construct within the statement. If we created a query using -only the second ``a2`` alias, the name would come out as ``addresses_1``. The -generation of the names is also *deterministic*, meaning the same SQLAlchemy -statement construct will produce the identical SQL string each time it is -rendered for a particular dialect. - -Since on the outside, we refer to the alias using the :class:`_expression.Alias` construct -itself, we don't need to be concerned about the generated name. However, for -the purposes of debugging, it can be specified by passing a string name -to the :meth:`_expression.FromClause.alias` method:: - - >>> a1 = addresses.alias('a1') - -SELECT-oriented constructs which extend from :class:`_expression.SelectBase` may be turned -into aliased subqueries using the :meth:`_expression.SelectBase.subquery` method, which -produces a :class:`.Subquery` construct; for ease of use, there is also a -:meth:`_expression.SelectBase.alias` method that is synonymous with -:meth:`_expression.SelectBase.subquery`. Like :class:`_expression.Alias`, :class:`.Subquery` is -also a :class:`_expression.FromClause` object that may be part of any enclosing SELECT -using the same techniques one would use for a :class:`_expression.Alias`. - -We can self-join the ``users`` table back to the :func:`_expression.select` we've created -by making :class:`.Subquery` of the entire statement: - -.. sourcecode:: pycon+sql - - >>> address_subq = s.subquery() - >>> s = select(users.c.name).where(users.c.id == address_subq.c.id) - >>> conn.execute(s).fetchall() - {opensql}SELECT users.name - FROM users, - (SELECT users.id AS id, users.name AS name, users.fullname AS fullname - FROM users, addresses AS addresses_1, addresses AS addresses_2 - WHERE users.id = addresses_1.user_id AND users.id = addresses_2.user_id - AND addresses_1.email_address = ? - AND addresses_2.email_address = ?) AS anon_1 - WHERE users.id = anon_1.id - [...] ('jack@msn.com', 'jack@yahoo.com') - {stop}[(u'jack',)] - -.. versionchanged:: 1.4 Added the :class:`.Subquery` object and created more of a - separation between an "alias" of a FROM clause and a named subquery of a - SELECT. See :ref:`change_4617`. - -Using Joins -=========== - -We're halfway along to being able to construct any SELECT expression. The next -cornerstone of the SELECT is the JOIN expression. We've already been doing -joins in our examples, by just placing two tables in either the columns clause -or the where clause of the :func:`_expression.select` construct. But if we want to make a -real "JOIN" or "OUTERJOIN" construct, we use the :meth:`_expression.FromClause.join` and -:meth:`_expression.FromClause.outerjoin` methods, most commonly accessed from the left table in the -join: - -.. sourcecode:: pycon+sql - - >>> print(users.join(addresses)) - users JOIN addresses ON users.id = addresses.user_id - -The alert reader will see more surprises; SQLAlchemy figured out how to JOIN -the two tables ! The ON condition of the join, as it's called, was -automatically generated based on the :class:`~sqlalchemy.schema.ForeignKey` -object which we placed on the ``addresses`` table way at the beginning of this -tutorial. Already the ``join()`` construct is looking like a much better way -to join tables. - -Of course you can join on whatever expression you want, such as if we want to -join on all users who use the same name in their email address as their -username: - -.. sourcecode:: pycon+sql - - >>> print(users.join(addresses, - ... addresses.c.email_address.like(users.c.name + '%') - ... ) - ... ) - users JOIN addresses ON addresses.email_address LIKE users.name || :name_1 - -When we create a :func:`_expression.select` construct, SQLAlchemy looks around at the -tables we've mentioned and then places them in the FROM clause of the -statement. When we use JOINs however, we know what FROM clause we want, so -here we make use of the :meth:`_expression.Select.select_from` method: - -.. sourcecode:: pycon+sql - - >>> s = select(users.c.fullname).select_from( - ... users.join(addresses, - ... addresses.c.email_address.like(users.c.name + '%')) - ... ) - {sql}>>> conn.execute(s).fetchall() - SELECT users.fullname - FROM users JOIN addresses ON addresses.email_address LIKE users.name || ? - [...] ('%',) - {stop}[(u'Jack Jones',), (u'Jack Jones',), (u'Wendy Williams',)] - -The :meth:`_expression.FromClause.outerjoin` method creates ``LEFT OUTER JOIN`` constructs, -and is used in the same way as :meth:`_expression.FromClause.join`: - -.. sourcecode:: pycon+sql - - >>> s = select(users.c.fullname).select_from(users.outerjoin(addresses)) - >>> print(s) - SELECT users.fullname - FROM users - LEFT OUTER JOIN addresses ON users.id = addresses.user_id - -That's the output ``outerjoin()`` produces, unless, of course, you're stuck in -a gig using Oracle prior to version 9, and you've set up your engine (which -would be using ``OracleDialect``) to use Oracle-specific SQL: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.dialects.oracle import dialect as OracleDialect - >>> print(s.compile(dialect=OracleDialect(use_ansi=False))) - SELECT users.fullname - FROM users, addresses - WHERE users.id = addresses.user_id(+) - -If you don't know what that SQL means, don't worry ! The secret tribe of -Oracle DBAs don't want their black magic being found out ;). - -.. seealso:: - - :func:`_expression.join` - - :func:`_expression.outerjoin` - - :class:`_expression.Join` - -Common Table Expressions (CTE) -============================== - -Common table expressions are now supported by every major database, including -modern MySQL, MariaDB, SQLite, PostgreSQL, Oracle and MS SQL Server. SQLAlchemy -supports this construct via the :class:`_expression.CTE` object, which one -typically acquires using the :meth:`_expression.Select.cte` method on a -:class:`_expression.Select` construct: - - -.. sourcecode:: pycon+sql - - >>> users_cte = select(users.c.id, users.c.name).where(users.c.name == 'wendy').cte() - >>> stmt = select(addresses).where(addresses.c.user_id == users_cte.c.id).order_by(addresses.c.id) - >>> conn.execute(stmt).fetchall() - {opensql}WITH anon_1 AS - (SELECT users.id AS id, users.name AS name - FROM users - WHERE users.name = ?) - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses, anon_1 - WHERE addresses.user_id = anon_1.id ORDER BY addresses.id - [...] ('wendy',) - {stop}[(3, 2, 'www@www.org'), (4, 2, 'wendy@aol.com')] - -The CTE construct is a great way to provide a source of rows that is -semantically similar to using a subquery, but with a much simpler format -where the source of rows is neatly tucked away at the top of the query -where it can be referenced anywhere in the main statement like a regular -table. - -When we construct a :class:`_expression.CTE` object, we make use of it like -any other table in the statement. However instead of being added to the -FROM clause as a subquery, it comes out on top, which has the additional -benefit of not causing surprise cartesian products. - -The RECURSIVE format of CTE is available when one uses the -:paramref:`_expression.Select.cte.recursive` parameter. A recursive -CTE typically requires that we are linking to ourselves as an alias. -The general form of this kind of operation involves a UNION of the -original CTE against itself. Noting that our example tables are not -well suited to producing an actually useful query with this feature, -this form looks like: - - -.. sourcecode:: pycon+sql - - >>> users_cte = select(users.c.id, users.c.name).cte(recursive=True) - >>> users_recursive = users_cte.alias() - >>> users_cte = users_cte.union(select(users.c.id, users.c.name).where(users.c.id > users_recursive.c.id)) - >>> stmt = select(addresses).where(addresses.c.user_id == users_cte.c.id).order_by(addresses.c.id) - >>> conn.execute(stmt).fetchall() - {opensql}WITH RECURSIVE anon_1(id, name) AS - (SELECT users.id AS id, users.name AS name - FROM users UNION SELECT users.id AS id, users.name AS name - FROM users, anon_1 AS anon_2 - WHERE users.id > anon_2.id) - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses, anon_1 - WHERE addresses.user_id = anon_1.id ORDER BY addresses.id - [...] () - {stop}[(1, 1, 'jack@yahoo.com'), (2, 1, 'jack@msn.com'), (3, 2, 'www@www.org'), (4, 2, 'wendy@aol.com')] - - -Everything Else -=============== - -The concepts of creating SQL expressions have been introduced. What's left are -more variants of the same themes. So now we'll catalog the rest of the -important things we'll need to know. - -.. _coretutorial_bind_param: - -Bind Parameter Objects ----------------------- - -Throughout all these examples, SQLAlchemy is busy creating bind parameters -wherever literal expressions occur. You can also specify your own bind -parameters with your own names, and use the same statement repeatedly. -The :func:`.bindparam` construct is used to produce a bound parameter -with a given name. While SQLAlchemy always refers to bound parameters by -name on the API side, the -database dialect converts to the appropriate named or positional style -at execution time, as here where it converts to positional for SQLite: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import bindparam - >>> s = users.select().where(users.c.name == bindparam('username')) - {sql}>>> conn.execute(s, {"username": "wendy"}).fetchall() - SELECT users.id, users.name, users.fullname - FROM users - WHERE users.name = ? - [...] ('wendy',) - {stop}[(2, u'wendy', u'Wendy Williams')] - -Another important aspect of :func:`.bindparam` is that it may be assigned a -type. The type of the bind parameter will determine its behavior within -expressions and also how the data bound to it is processed before being sent -off to the database: - -.. sourcecode:: pycon+sql - - >>> s = users.select().where(users.c.name.like(bindparam('username', type_=String) + text("'%'"))) - {sql}>>> conn.execute(s, {"username": "wendy"}).fetchall() - SELECT users.id, users.name, users.fullname - FROM users - WHERE users.name LIKE ? || '%' - [...] ('wendy',) - {stop}[(2, u'wendy', u'Wendy Williams')] - - -:func:`.bindparam` constructs of the same name can also be used multiple times, where only a -single named value is needed in the execute parameters: - -.. sourcecode:: pycon+sql - - >>> s = select(users, addresses).\ - ... where( - ... or_( - ... users.c.name.like( - ... bindparam('name', type_=String) + text("'%'")), - ... addresses.c.email_address.like( - ... bindparam('name', type_=String) + text("'@%'")) - ... ) - ... ).\ - ... select_from(users.outerjoin(addresses)).\ - ... order_by(addresses.c.id) - {sql}>>> conn.execute(s, {"name": "jack"}).fetchall() - SELECT users.id, users.name, users.fullname, addresses.id AS id_1, - addresses.user_id, addresses.email_address - FROM users LEFT OUTER JOIN addresses ON users.id = addresses.user_id - WHERE users.name LIKE ? || '%' OR addresses.email_address LIKE ? || '@%' - ORDER BY addresses.id - [...] ('jack', 'jack') - {stop}[(1, u'jack', u'Jack Jones', 1, 1, u'jack@yahoo.com'), (1, u'jack', u'Jack Jones', 2, 1, u'jack@msn.com')] - -.. seealso:: - - :func:`.bindparam` - -.. _coretutorial_functions: - -Functions ---------- - -SQL functions are created using the :data:`~.expression.func` keyword, which -generates functions using attribute access: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import func - >>> print(func.now()) - now() - - >>> print(func.concat('x', 'y')) - concat(:concat_1, :concat_2) - -By "generates", we mean that **any** SQL function is created based on the word -you choose:: - - >>> print(func.xyz_my_goofy_function()) - xyz_my_goofy_function() - -Certain function names are known by SQLAlchemy, allowing special behavioral -rules to be applied. Some for example are "ANSI" functions, which mean they -don't get the parenthesis added after them, such as CURRENT_TIMESTAMP: - -.. sourcecode:: pycon+sql - - >>> print(func.current_timestamp()) - CURRENT_TIMESTAMP - -A function, like any other column expression, has a type, which indicates the -type of expression as well as how SQLAlchemy will interpret result columns -that are returned from this expression. The default type used for an -arbitrary function name derived from :attr:`.func` is simply a "null" datatype. -However, in order for the column expression generated by the function to -have type-specific operator behavior as well as result-set behaviors, such -as date and numeric coercions, the type may need to be specified explicitly:: - - stmt = select(func.date(some_table.c.date_string, type_=Date)) - - -Functions are most typically used in the columns clause of a select statement, -and can also be labeled as well as given a type. Labeling a function is -recommended so that the result can be targeted in a result row based on a -string name, and assigning it a type is required when you need result-set -processing to occur, such as for Unicode conversion and date conversions. -Below, we use the result function ``scalar()`` to just read the first column -of the first row and then close the result; the label, even though present, is -not important in this case: - -.. sourcecode:: pycon+sql - - >>> conn.execute( - ... select( - ... func.max(addresses.c.email_address, type_=String). - ... label('maxemail') - ... ) - ... ).scalar() - {opensql}SELECT max(addresses.email_address) AS maxemail - FROM addresses - [...] () - {stop}u'www@www.org' - -Databases such as PostgreSQL and Oracle which support functions that return -whole result sets can be assembled into selectable units, which can be used in -statements. Such as, a database function ``calculate()`` which takes the -parameters ``x`` and ``y``, and returns three columns which we'd like to name -``q``, ``z`` and ``r``, we can construct using "lexical" column objects as -well as bind parameters: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import column - >>> calculate = select(column('q'), column('z'), column('r')).\ - ... select_from( - ... func.calculate( - ... bindparam('x'), - ... bindparam('y') - ... ) - ... ) - >>> calc = calculate.alias() - >>> print(select(users).where(users.c.id > calc.c.z)) - SELECT users.id, users.name, users.fullname - FROM users, (SELECT q, z, r - FROM calculate(:x, :y)) AS anon_1 - WHERE users.id > anon_1.z - -If we wanted to use our ``calculate`` statement twice with different bind -parameters, the :func:`~sqlalchemy.sql.expression.ClauseElement.unique_params` -function will create copies for us, and mark the bind parameters as "unique" -so that conflicting names are isolated. Note we also make two separate aliases -of our selectable: - -.. sourcecode:: pycon+sql - - >>> calc1 = calculate.alias('c1').unique_params(x=17, y=45) - >>> calc2 = calculate.alias('c2').unique_params(x=5, y=12) - >>> s = select(users).\ - ... where(users.c.id.between(calc1.c.z, calc2.c.z)) - >>> print(s) - SELECT users.id, users.name, users.fullname - FROM users, - (SELECT q, z, r FROM calculate(:x_1, :y_1)) AS c1, - (SELECT q, z, r FROM calculate(:x_2, :y_2)) AS c2 - WHERE users.id BETWEEN c1.z AND c2.z - - >>> s.compile().params # doctest: +SKIP - {u'x_2': 5, u'y_2': 12, u'y_1': 45, u'x_1': 17} - -.. seealso:: - - :data:`.func` - -.. _window_functions: - -Window Functions ----------------- - -Any :class:`.FunctionElement`, including functions generated by -:data:`~.expression.func`, can be turned into a "window function", that is an -OVER clause, using the :meth:`.FunctionElement.over` method:: - - >>> s = select( - ... users.c.id, - ... func.row_number().over(order_by=users.c.name) - ... ) - >>> print(s) - SELECT users.id, row_number() OVER (ORDER BY users.name) AS anon_1 - FROM users - -:meth:`.FunctionElement.over` also supports range specification using -either the :paramref:`.expression.over.rows` or -:paramref:`.expression.over.range` parameters:: - - >>> s = select( - ... users.c.id, - ... func.row_number().over( - ... order_by=users.c.name, - ... rows=(-2, None)) - ... ) - >>> print(s) - SELECT users.id, row_number() OVER - (ORDER BY users.name ROWS BETWEEN :param_1 PRECEDING AND UNBOUNDED FOLLOWING) AS anon_1 - FROM users - -:paramref:`.expression.over.rows` and :paramref:`.expression.over.range` each -accept a two-tuple which contains a combination of negative and positive -integers for ranges, zero to indicate "CURRENT ROW" and ``None`` to -indicate "UNBOUNDED". See the examples at :func:`.over` for more detail. - -.. versionadded:: 1.1 support for "rows" and "range" specification for - window functions - -.. seealso:: - - :func:`.over` - - :meth:`.FunctionElement.over` - -.. _coretutorial_casts: - -Data Casts and Type Coercion ------------------------------ - -In SQL, we often need to indicate the datatype of an element explicitly, or -we need to convert between one datatype and another within a SQL statement. -The CAST SQL function performs this. In SQLAlchemy, the :func:`.cast` function -renders the SQL CAST keyword. It accepts a column expression and a data type -object as arguments: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy import cast - >>> s = select(cast(users.c.id, String)) - >>> conn.execute(s).fetchall() - {opensql}SELECT CAST(users.id AS VARCHAR) AS id - FROM users - [...] () - {stop}[('1',), ('2',)] - -The :func:`.cast` function is used not just when converting between datatypes, -but also in cases where the database needs to -know that some particular value should be considered to be of a particular -datatype within an expression. - -The :func:`.cast` function also tells SQLAlchemy itself that an expression -should be treated as a particular type as well. The datatype of an expression -directly impacts the behavior of Python operators upon that object, such as how -the ``+`` operator may indicate integer addition or string concatenation, and -it also impacts how a literal Python value is transformed or handled before -being passed to the database as well as how result values of that expression -should be transformed or handled. - -Sometimes there is the need to have SQLAlchemy know the datatype of an -expression, for all the reasons mentioned above, but to not render the CAST -expression itself on the SQL side, where it may interfere with a SQL operation -that already works without it. For this fairly common use case there is -another function :func:`.type_coerce` which is closely related to -:func:`.cast`, in that it sets up a Python expression as having a specific SQL -database type, but does not render the ``CAST`` keyword or datatype on the -database side. :func:`.type_coerce` is particularly important when dealing -with the :class:`_types.JSON` datatype, which typically has an intricate -relationship with string-oriented datatypes on different platforms and -may not even be an explicit datatype, such as on SQLite and MariaDB. -Below, we use :func:`.type_coerce` to deliver a Python structure as a JSON -string into one of MySQL's JSON functions: - -.. sourcecode:: pycon+sql - - >>> import json - >>> from sqlalchemy import JSON - >>> from sqlalchemy import type_coerce - >>> from sqlalchemy.dialects import mysql - >>> s = select( - ... type_coerce( - ... {'some_key': {'foo': 'bar'}}, JSON - ... )['some_key'] - ... ) - >>> print(s.compile(dialect=mysql.dialect())) - SELECT JSON_EXTRACT(%s, %s) AS anon_1 - -Above, MySQL's ``JSON_EXTRACT`` SQL function was invoked -because we used :func:`.type_coerce` to indicate that our Python dictionary -should be treated as :class:`_types.JSON`. The Python ``__getitem__`` -operator, ``['some_key']`` in this case, became available as a result and -allowed a ``JSON_EXTRACT`` path expression (not shown, however in this -case it would ultimately be ``'$."some_key"'``) to be rendered. - -Unions and Other Set Operations -------------------------------- - -Unions come in two flavors, UNION and UNION ALL, which are available via -module level functions :func:`_expression.union` and -:func:`_expression.union_all`: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import union - >>> u = union( - ... addresses.select(). - ... where(addresses.c.email_address == 'foo@bar.com'), - ... addresses.select(). - ... where(addresses.c.email_address.like('%@yahoo.com')), - ... ).order_by(addresses.c.email_address) - - {sql}>>> conn.execute(u).fetchall() - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address = ? - UNION - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address LIKE ? ORDER BY email_address - [...] ('foo@bar.com', '%@yahoo.com') - {stop}[(1, 1, u'jack@yahoo.com')] - -Also available, though not supported on all databases, are -:func:`_expression.intersect`, -:func:`_expression.intersect_all`, -:func:`_expression.except_`, and :func:`_expression.except_all`: - -.. sourcecode:: pycon+sql - - >>> from sqlalchemy.sql import except_ - >>> u = except_( - ... addresses.select(). - ... where(addresses.c.email_address.like('%@%.com')), - ... addresses.select(). - ... where(addresses.c.email_address.like('%@msn.com')) - ... ) - - {sql}>>> conn.execute(u).fetchall() - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address LIKE ? - EXCEPT - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address LIKE ? - [...] ('%@%.com', '%@msn.com') - {stop}[(1, 1, u'jack@yahoo.com'), (4, 2, u'wendy@aol.com')] - -A common issue with so-called "compound" selectables arises due to the fact -that they nest with parenthesis. SQLite in particular doesn't like a statement -that starts with parenthesis. So when nesting a "compound" inside a -"compound", it's often necessary to apply ``.subquery().select()`` to the first -element of the outermost compound, if that element is also a compound. For -example, to nest a "union" and a "select" inside of "except\_", SQLite will -want the "union" to be stated as a subquery: - -.. sourcecode:: pycon+sql - - >>> u = except_( - ... union( - ... addresses.select(). - ... where(addresses.c.email_address.like('%@yahoo.com')), - ... addresses.select(). - ... where(addresses.c.email_address.like('%@msn.com')) - ... ).subquery().select(), # apply subquery here - ... addresses.select().where(addresses.c.email_address.like('%@msn.com')) - ... ) - {sql}>>> conn.execute(u).fetchall() - SELECT anon_1.id, anon_1.user_id, anon_1.email_address - FROM (SELECT addresses.id AS id, addresses.user_id AS user_id, - addresses.email_address AS email_address - FROM addresses - WHERE addresses.email_address LIKE ? - UNION - SELECT addresses.id AS id, - addresses.user_id AS user_id, - addresses.email_address AS email_address - FROM addresses - WHERE addresses.email_address LIKE ?) AS anon_1 - EXCEPT - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address LIKE ? - [...] ('%@yahoo.com', '%@msn.com', '%@msn.com') - {stop}[(1, 1, u'jack@yahoo.com')] - -.. seealso:: - - :func:`_expression.union` - - :func:`_expression.union_all` - - :func:`_expression.intersect` - - :func:`_expression.intersect_all` - - :func:`.except_` - - :func:`_expression.except_all` - -Ordering Unions -^^^^^^^^^^^^^^^ - -UNION and other set constructs have a special case when it comes to ordering -the results. As the UNION consists of several SELECT statements, to ORDER the -whole result usually requires that an ORDER BY clause refer to column names but -not specific tables. As in the previous examples, we used -``.order_by(addresses.c.email_address)`` but SQLAlchemy rendered the ORDER BY -without using the table name. A generalized way to apply ORDER BY to a union -is also to refer to the :attr:`_selectable.CompoundSelect.selected_columns` collection in -order to access the column expressions which are synonymous with the columns -selected from the first SELECT; the SQLAlchemy compiler will ensure these will -be rendered without table names:: - - >>> u = union( - ... addresses.select(). - ... where(addresses.c.email_address == 'foo@bar.com'), - ... addresses.select(). - ... where(addresses.c.email_address.like('%@yahoo.com')), - ... ) - >>> u = u.order_by(u.selected_columns.email_address) - >>> print(u) - SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address = :email_address_1 - UNION SELECT addresses.id, addresses.user_id, addresses.email_address - FROM addresses - WHERE addresses.email_address LIKE :email_address_2 ORDER BY email_address - - -.. _scalar_selects: - -Scalar Selects --------------- - -A scalar select is a SELECT that returns exactly one row and one -column. It can then be used as a column expression. A scalar select -is often a :term:`correlated subquery`, which relies upon the enclosing -SELECT statement in order to acquire at least one of its FROM clauses. - -The :func:`_expression.select` construct can be modified to act as a -column expression by calling either the :meth:`_expression.SelectBase.scalar_subquery` -or :meth:`_expression.SelectBase.label` method: - -.. sourcecode:: pycon+sql - - >>> subq = select(func.count(addresses.c.id)).\ - ... where(users.c.id == addresses.c.user_id).\ - ... scalar_subquery() - -The above construct is now a :class:`_expression.ScalarSelect` object, -which is an adapter around the original :class:`.~expression.Select` -object; it participates within the :class:`_expression.ColumnElement` -family of expression constructs. We can place this construct the same as any -other column within another :func:`_expression.select`: - -.. sourcecode:: pycon+sql - - >>> conn.execute(select(users.c.name, subq)).fetchall() - {opensql}SELECT users.name, (SELECT count(addresses.id) AS count_1 - FROM addresses - WHERE users.id = addresses.user_id) AS anon_1 - FROM users - [...] () - {stop}[(u'jack', 2), (u'wendy', 2)] - -To apply a non-anonymous column name to our scalar select, we create -it using :meth:`_expression.SelectBase.label` instead: - -.. sourcecode:: pycon+sql - - >>> subq = select(func.count(addresses.c.id)).\ - ... where(users.c.id == addresses.c.user_id).\ - ... label("address_count") - >>> conn.execute(select(users.c.name, subq)).fetchall() - {opensql}SELECT users.name, (SELECT count(addresses.id) AS count_1 - FROM addresses - WHERE users.id = addresses.user_id) AS address_count - FROM users - [...] () - {stop}[(u'jack', 2), (u'wendy', 2)] - -.. seealso:: - - :meth:`_expression.Select.scalar_subquery` - - :meth:`_expression.Select.label` - -.. _correlated_subqueries: - -Correlated Subqueries ---------------------- - -In the examples on :ref:`scalar_selects`, the FROM clause of each embedded -select did not contain the ``users`` table in its FROM clause. This is because -SQLAlchemy automatically :term:`correlates` embedded FROM objects to that -of an enclosing query, if present, and if the inner SELECT statement would -still have at least one FROM clause of its own. For example: - -.. sourcecode:: pycon+sql - - >>> stmt = select(addresses.c.user_id).\ - ... where(addresses.c.user_id == users.c.id).\ - ... where(addresses.c.email_address == 'jack@yahoo.com') - >>> enclosing_stmt = select(users.c.name).\ - ... where(users.c.id == stmt.scalar_subquery()) - >>> conn.execute(enclosing_stmt).fetchall() - {opensql}SELECT users.name - FROM users - WHERE users.id = (SELECT addresses.user_id - FROM addresses - WHERE addresses.user_id = users.id - AND addresses.email_address = ?) - [...] ('jack@yahoo.com',) - {stop}[(u'jack',)] - -Auto-correlation will usually do what's expected, however it can also be controlled. -For example, if we wanted a statement to correlate only to the ``addresses`` table -but not the ``users`` table, even if both were present in the enclosing SELECT, -we use the :meth:`_expression.Select.correlate` method to specify those FROM clauses that -may be correlated: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.id).\ - ... where(users.c.id == addresses.c.user_id).\ - ... where(users.c.name == 'jack').\ - ... correlate(addresses) - >>> enclosing_stmt = select( - ... users.c.name, addresses.c.email_address).\ - ... select_from(users.join(addresses)).\ - ... where(users.c.id == stmt.scalar_subquery()) - >>> conn.execute(enclosing_stmt).fetchall() - {opensql}SELECT users.name, addresses.email_address - FROM users JOIN addresses ON users.id = addresses.user_id - WHERE users.id = (SELECT users.id - FROM users - WHERE users.id = addresses.user_id AND users.name = ?) - [...] ('jack',) - {stop}[(u'jack', u'jack@yahoo.com'), (u'jack', u'jack@msn.com')] - -To entirely disable a statement from correlating, we can pass ``None`` -as the argument: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.id).\ - ... where(users.c.name == 'wendy').\ - ... correlate(None) - >>> enclosing_stmt = select(users.c.name).\ - ... where(users.c.id == stmt.scalar_subquery()) - >>> conn.execute(enclosing_stmt).fetchall() - {opensql}SELECT users.name - FROM users - WHERE users.id = (SELECT users.id - FROM users - WHERE users.name = ?) - [...] ('wendy',) - {stop}[(u'wendy',)] - -We can also control correlation via exclusion, using the :meth:`_expression.Select.correlate_except` -method. Such as, we can write our SELECT for the ``users`` table -by telling it to correlate all FROM clauses except for ``users``: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.id).\ - ... where(users.c.id == addresses.c.user_id).\ - ... where(users.c.name == 'jack').\ - ... correlate_except(users) - >>> enclosing_stmt = select( - ... users.c.name, addresses.c.email_address).\ - ... select_from(users.join(addresses)).\ - ... where(users.c.id == stmt.scalar_subquery()) - >>> conn.execute(enclosing_stmt).fetchall() - {opensql}SELECT users.name, addresses.email_address - FROM users JOIN addresses ON users.id = addresses.user_id - WHERE users.id = (SELECT users.id - FROM users - WHERE users.id = addresses.user_id AND users.name = ?) - [...] ('jack',) - {stop}[(u'jack', u'jack@yahoo.com'), (u'jack', u'jack@msn.com')] - -.. _lateral_selects: - -LATERAL correlation -^^^^^^^^^^^^^^^^^^^ - -LATERAL correlation is a special sub-category of SQL correlation which -allows a selectable unit to refer to another selectable unit within a -single FROM clause. This is an extremely special use case which, while -part of the SQL standard, is only known to be supported by recent -versions of PostgreSQL. - -Normally, if a SELECT statement refers to -``table1 JOIN (some SELECT) AS subquery`` in its FROM clause, the subquery -on the right side may not refer to the "table1" expression from the left side; -correlation may only refer to a table that is part of another SELECT that -entirely encloses this SELECT. The LATERAL keyword allows us to turn this -behavior around, allowing an expression such as: - -.. sourcecode:: sql - - SELECT people.people_id, people.age, people.name - FROM people JOIN LATERAL (SELECT books.book_id AS book_id - FROM books WHERE books.owner_id = people.people_id) - AS book_subq ON true - -Where above, the right side of the JOIN contains a subquery that refers not -just to the "books" table but also the "people" table, correlating -to the left side of the JOIN. SQLAlchemy Core supports a statement -like the above using the :meth:`_expression.Select.lateral` method as follows:: - - >>> from sqlalchemy import table, column, select, true - >>> people = table('people', column('people_id'), column('age'), column('name')) - >>> books = table('books', column('book_id'), column('owner_id')) - >>> subq = select(books.c.book_id).\ - ... where(books.c.owner_id == people.c.people_id).lateral("book_subq") - >>> print(select(people).select_from(people.join(subq, true()))) - SELECT people.people_id, people.age, people.name - FROM people JOIN LATERAL (SELECT books.book_id AS book_id - FROM books WHERE books.owner_id = people.people_id) - AS book_subq ON true - -Above, we can see that the :meth:`_expression.Select.lateral` method acts a lot like -the :meth:`_expression.Select.alias` method, including that we can specify an optional -name. However the construct is the :class:`_expression.Lateral` construct instead of -an :class:`_expression.Alias` which provides for the LATERAL keyword as well as special -instructions to allow correlation from inside the FROM clause of the -enclosing statement. - -The :meth:`_expression.Select.lateral` method interacts normally with the -:meth:`_expression.Select.correlate` and :meth:`_expression.Select.correlate_except` methods, except -that the correlation rules also apply to any other tables present in the -enclosing statement's FROM clause. Correlation is "automatic" to these -tables by default, is explicit if the table is specified to -:meth:`_expression.Select.correlate`, and is explicit to all tables except those -specified to :meth:`_expression.Select.correlate_except`. - - -.. versionadded:: 1.1 - - Support for the LATERAL keyword and lateral correlation. - -.. seealso:: - - :class:`_expression.Lateral` - - :meth:`_expression.Select.lateral` - - -.. _core_tutorial_ordering: - -Ordering, Grouping, Limiting, Offset...ing... ---------------------------------------------- - -Ordering is done by passing column expressions to the -:meth:`_expression.SelectBase.order_by` method: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name).order_by(users.c.name) - >>> conn.execute(stmt).fetchall() - {opensql}SELECT users.name - FROM users ORDER BY users.name - [...] () - {stop}[(u'jack',), (u'wendy',)] - -Ascending or descending can be controlled using the :meth:`_expression.ColumnElement.asc` -and :meth:`_expression.ColumnElement.desc` modifiers: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name).order_by(users.c.name.desc()) - >>> conn.execute(stmt).fetchall() - {opensql}SELECT users.name - FROM users ORDER BY users.name DESC - [...] () - {stop}[(u'wendy',), (u'jack',)] - -Grouping refers to the GROUP BY clause, and is usually used in conjunction -with aggregate functions to establish groups of rows to be aggregated. -This is provided via the :meth:`_expression.SelectBase.group_by` method: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name, func.count(addresses.c.id)).\ - ... select_from(users.join(addresses)).\ - ... group_by(users.c.name) - >>> conn.execute(stmt).fetchall() - {opensql}SELECT users.name, count(addresses.id) AS count_1 - FROM users JOIN addresses - ON users.id = addresses.user_id - GROUP BY users.name - [...] () - {stop}[(u'jack', 2), (u'wendy', 2)] - -See also :ref:`sqlexpression_order_by_label` for an important technique -of ordering or grouping by a string column name. - -HAVING can be used to filter results on an aggregate value, after GROUP BY has -been applied. It's available here via the :meth:`_expression.Select.having` -method: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name, func.count(addresses.c.id)).\ - ... select_from(users.join(addresses)).\ - ... group_by(users.c.name).\ - ... having(func.length(users.c.name) > 4) - >>> conn.execute(stmt).fetchall() - {opensql}SELECT users.name, count(addresses.id) AS count_1 - FROM users JOIN addresses - ON users.id = addresses.user_id - GROUP BY users.name - HAVING length(users.name) > ? - [...] (4,) - {stop}[(u'wendy', 2)] - -A common system of dealing with duplicates in composed SELECT statements -is the DISTINCT modifier. A simple DISTINCT clause can be added using the -:meth:`_expression.Select.distinct` method: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name).\ - ... where(addresses.c.email_address. - ... contains(users.c.name)).\ - ... distinct() - >>> conn.execute(stmt).fetchall() - {opensql}SELECT DISTINCT users.name - FROM users, addresses - WHERE (addresses.email_address LIKE '%' || users.name || '%') - [...] () - {stop}[(u'jack',), (u'wendy',)] - -Most database backends support a system of limiting how many rows -are returned, and the majority also feature a means of starting to return -rows after a given "offset". While common backends like PostgreSQL, -MySQL and SQLite support LIMIT and OFFSET keywords, other backends -need to refer to more esoteric features such as "window functions" -and row ids to achieve the same effect. The :meth:`_expression.Select.limit` -and :meth:`_expression.Select.offset` methods provide an easy abstraction -into the current backend's methodology: - -.. sourcecode:: pycon+sql - - >>> stmt = select(users.c.name, addresses.c.email_address).\ - ... select_from(users.join(addresses)).\ - ... limit(1).offset(1) - >>> conn.execute(stmt).fetchall() - {opensql}SELECT users.name, addresses.email_address - FROM users JOIN addresses ON users.id = addresses.user_id - LIMIT ? OFFSET ? - [...] (1, 1) - {stop}[(u'jack', u'jack@msn.com')] - - -.. _inserts_and_updates: - -Inserts, Updates and Deletes -============================ - -We've seen :meth:`_expression.TableClause.insert` demonstrated -earlier in this tutorial. Where :meth:`_expression.TableClause.insert` -produces INSERT, the :meth:`_expression.TableClause.update` -method produces UPDATE. Both of these constructs feature -a method called :meth:`~.ValuesBase.values` which specifies -the VALUES or SET clause of the statement. - -The :meth:`~.ValuesBase.values` method accommodates any column expression -as a value: - -.. sourcecode:: pycon+sql - - >>> stmt = users.update().\ - ... values(fullname="Fullname: " + users.c.name) - >>> conn.execute(stmt) - {opensql}UPDATE users SET fullname=(? || users.name) - [...] ('Fullname: ',) - COMMIT - {stop} - -When using :meth:`_expression.TableClause.insert` or :meth:`_expression.TableClause.update` -in an "execute many" context, we may also want to specify named -bound parameters which we can refer to in the argument list. -The two constructs will automatically generate bound placeholders -for any column names passed in the dictionaries sent to -:meth:`_engine.Connection.execute` at execution time. However, if we -wish to use explicitly targeted named parameters with composed expressions, -we need to use the :func:`_expression.bindparam` construct. -When using :func:`_expression.bindparam` with -:meth:`_expression.TableClause.insert` or :meth:`_expression.TableClause.update`, -the names of the table's columns themselves are reserved for the -"automatic" generation of bind names. We can combine the usage -of implicitly available bind names and explicitly named parameters -as in the example below: - -.. sourcecode:: pycon+sql - - >>> stmt = users.insert().\ - ... values(name=bindparam('_name') + " .. name") - >>> conn.execute(stmt, [ - ... {'id':4, '_name':'name1'}, - ... {'id':5, '_name':'name2'}, - ... {'id':6, '_name':'name3'}, - ... ]) - {opensql}INSERT INTO users (id, name) VALUES (?, (? || ?)) - [...] ((4, 'name1', ' .. name'), (5, 'name2', ' .. name'), (6, 'name3', ' .. name')) - COMMIT - - -An UPDATE statement is emitted using the :meth:`_expression.TableClause.update` construct. This -works much like an INSERT, except there is an additional WHERE clause -that can be specified: - -.. sourcecode:: pycon+sql - - >>> stmt = users.update().\ - ... where(users.c.name == 'jack').\ - ... values(name='ed') - - >>> conn.execute(stmt) - {opensql}UPDATE users SET name=? WHERE users.name = ? - [...] ('ed', 'jack') - COMMIT - {stop} - -When using :meth:`_expression.TableClause.update` in an "executemany" context, -we may wish to also use explicitly named bound parameters in the -WHERE clause. Again, :func:`_expression.bindparam` is the construct -used to achieve this: - -.. sourcecode:: pycon+sql - - >>> stmt = users.update().\ - ... where(users.c.name == bindparam('oldname')).\ - ... values(name=bindparam('newname')) - >>> conn.execute(stmt, [ - ... {'oldname':'jack', 'newname':'ed'}, - ... {'oldname':'wendy', 'newname':'mary'}, - ... {'oldname':'jim', 'newname':'jake'}, - ... ]) - {opensql}UPDATE users SET name=? WHERE users.name = ? - [...] (('ed', 'jack'), ('mary', 'wendy'), ('jake', 'jim')) - COMMIT - {stop} - -.. _tutorial_1x_correlated_updates: - -Correlated Updates ------------------- - -A correlated update lets you update a table using selection from another -table, or the same table; the SELECT statement is passed as a scalar -subquery using :meth:`_expression.Select.scalar_subquery`: - -.. sourcecode:: pycon+sql - - >>> stmt = select(addresses.c.email_address).\ - ... where(addresses.c.user_id == users.c.id).\ - ... limit(1) - >>> conn.execute(users.update().values(fullname=stmt.scalar_subquery())) - {opensql}UPDATE users SET fullname=(SELECT addresses.email_address - FROM addresses - WHERE addresses.user_id = users.id - LIMIT ? OFFSET ?) - [...] (1, 0) - COMMIT - {stop} - -.. _multi_table_updates: - -Multiple Table Updates ----------------------- - -The PostgreSQL, Microsoft SQL Server, and MySQL backends all support UPDATE statements -that refer to multiple tables. For PG and MSSQL, this is the "UPDATE FROM" syntax, -which updates one table at a time, but can reference additional tables in an additional -"FROM" clause that can then be referenced in the WHERE clause directly. On MySQL, -multiple tables can be embedded into a single UPDATE statement separated by a comma. -The SQLAlchemy :func:`_expression.update` construct supports both of these modes -implicitly, by specifying multiple tables in the WHERE clause:: - - stmt = users.update().\ - values(name='ed wood').\ - where(users.c.id == addresses.c.id).\ - where(addresses.c.email_address.startswith('ed%')) - conn.execute(stmt) - -The resulting SQL from the above statement would render as:: - - UPDATE users SET name=:name FROM addresses - WHERE users.id = addresses.id AND - addresses.email_address LIKE :email_address_1 || '%' - -When using MySQL, columns from each table can be assigned to in the -SET clause directly, using the dictionary form passed to :meth:`_expression.Update.values`:: - - stmt = users.update().\ - values({ - users.c.name:'ed wood', - addresses.c.email_address:'ed.wood@foo.com' - }).\ - where(users.c.id == addresses.c.id).\ - where(addresses.c.email_address.startswith('ed%')) - -The tables are referenced explicitly in the SET clause:: - - UPDATE users, addresses SET addresses.email_address=%s, - users.name=%s WHERE users.id = addresses.id - AND addresses.email_address LIKE concat(%s, '%') - -When the construct is used on a non-supporting database, the compiler -will raise ``NotImplementedError``. For convenience, when a statement -is printed as a string without specification of a dialect, the "string SQL" -compiler will be invoked which provides a non-working SQL representation of the -construct. - -.. _updates_order_parameters: - -Parameter-Ordered Updates -------------------------- - -The default behavior of the :func:`_expression.update` construct when rendering the SET -clauses is to render them using the column ordering given in the -originating :class:`_schema.Table` object. -This is an important behavior, since it means that the rendering of a -particular UPDATE statement with particular columns -will be rendered the same each time, which has an impact on query caching systems -that rely on the form of the statement, either client side or server side. -Since the parameters themselves are passed to the :meth:`_expression.Update.values` -method as Python dictionary keys, there is no other fixed ordering -available. - -However in some cases, the order of parameters rendered in the SET clause of an -UPDATE statement may need to be explicitly stated. The main example of this is -when using MySQL and providing updates to column values based on that of other -column values. The end result of the following statement:: - - UPDATE some_table SET x = y + 10, y = 20 - -Will have a different result than:: - - UPDATE some_table SET y = 20, x = y + 10 - -This because on MySQL, the individual SET clauses are fully evaluated on -a per-value basis, as opposed to on a per-row basis, and as each SET clause -is evaluated, the values embedded in the row are changing. - -To suit this specific use case, the -:meth:`_expression.update.ordered_values` method may be used. When using this method, -we supply a **series of 2-tuples** -as the argument to the method:: - - stmt = some_table.update().\ - ordered_values((some_table.c.y, 20), (some_table.c.x, some_table.c.y + 10)) - -The series of 2-tuples is essentially the same structure as a Python -dictionary, except that it explicitly suggests a specific ordering. Using the -above form, we are assured that the "y" column's SET clause will render first, -then the "x" column's SET clause. - -.. versionchanged:: 1.4 Added the :meth:`_expression.Update.ordered_values` method which - supersedes the :paramref:`_expression.update.preserve_parameter_order` flag that will - be removed in SQLAlchemy 2.0. - -.. seealso:: - - :ref:`mysql_insert_on_duplicate_key_update` - background on the MySQL - ``ON DUPLICATE KEY UPDATE`` clause and how to support parameter ordering. - -.. _deletes: - -Deletes -------- - -Finally, a delete. This is accomplished easily enough using the -:meth:`_expression.TableClause.delete` construct: - -.. sourcecode:: pycon+sql - - >>> conn.execute(addresses.delete()) - {opensql}DELETE FROM addresses - [...] () - COMMIT - {stop} - - >>> conn.execute(users.delete().where(users.c.name > 'm')) - {opensql}DELETE FROM users WHERE users.name > ? - [...] ('m',) - COMMIT - {stop} - -.. _multi_table_deletes: - -Multiple Table Deletes ----------------------- - -.. versionadded:: 1.2 - -The PostgreSQL, Microsoft SQL Server, and MySQL backends all support DELETE -statements that refer to multiple tables within the WHERE criteria. For PG -and MySQL, this is the "DELETE USING" syntax, and for SQL Server, it's a -"DELETE FROM" that refers to more than one table. The SQLAlchemy -:func:`_expression.delete` construct supports both of these modes -implicitly, by specifying multiple tables in the WHERE clause:: - - stmt = users.delete().\ - where(users.c.id == addresses.c.id).\ - where(addresses.c.email_address.startswith('ed%')) - conn.execute(stmt) - -On a PostgreSQL backend, the resulting SQL from the above statement would render as:: - - DELETE FROM users USING addresses - WHERE users.id = addresses.id - AND (addresses.email_address LIKE %(email_address_1)s || '%%') - -When the construct is used on a non-supporting database, the compiler -will raise ``NotImplementedError``. For convenience, when a statement -is printed as a string without specification of a dialect, the "string SQL" -compiler will be invoked which provides a non-working SQL representation of the -construct. - -Matched Row Counts ------------------- - -Both of :meth:`_expression.TableClause.update` and -:meth:`_expression.TableClause.delete` are associated with *matched row counts*. This is a -number indicating the number of rows that were matched by the WHERE clause. -Note that by "matched", this includes rows where no UPDATE actually took place. -The value is available as :attr:`_engine.CursorResult.rowcount`: - -.. sourcecode:: pycon+sql - - >>> result = conn.execute(users.delete()) - {opensql}DELETE FROM users - [...] () - COMMIT - {stop}>>> result.rowcount - 1 - -Further Reference -================= - -Expression Language Reference: :ref:`expression_api_toplevel` - -Database Metadata Reference: :ref:`metadata_toplevel` - -Engine Reference: :doc:`/core/engines` - -Connection Reference: :ref:`connections_toplevel` - -Types Reference: :ref:`types_toplevel` - - - -.. Setup code, not for display - - >>> conn.close() diff --git a/doc/build/index.rst b/doc/build/index.rst index 35005872f4..e363f581f1 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -21,7 +21,7 @@ SQLAlchemy Documentation :doc:`Overview ` | :ref:`Installation Guide ` | :doc:`Frequently Asked Questions ` | - :doc:`Migration from 1.3 ` | + :doc:`Migration from 1.4 ` | :doc:`Glossary ` | :doc:`Error Messages ` | :doc:`Changelog catalog ` @@ -35,31 +35,26 @@ SQLAlchemy Documentation Tutorials + .. the paragraph below for "sqlalchemy 2.0" seems to be too wide to be + easily readable. suggest some kind of layout change that can keep the + paragraph width more narrow even if there is just one row on the page. + .. container:: - **SQLAlchemy 1.4 / 2.0 Transitional** + **SQLAlchemy 2.0** - SQLAlchemy 2.0 is functionally available as part of SQLAlchemy 1.4, and integrates - Core and ORM working styles more closely than ever. The new tutorial introduces - both concepts in parallel. New users and those starting new projects should start here! + The SQLAlchemy 2.0 series represents a major rework of the classic 1.x + SQLAlchemy APIs that have evolved over more than 15 years. The + SQLAlchemy tutorial provides a holistic view of the library, integrating + Core and ORM features in a narrative style that is optimized towards + establihsing a solid understanding of the foundations upon which + SQLAlchemy is built on. The tutorials are recommended for all new users + as well as veterans of older SQLAlchemy versions alike. * :doc:`/tutorial/index` - SQLAlchemy 2.0's main tutorial * :doc:`Migrating to SQLAlchemy 2.0 ` - Complete background on migrating from 1.3 or 1.4 to 2.0 - - .. container:: - - **SQLAlchemy 1.x Releases** - - The 1.x Object Relational Tutorial and Core Tutorial are the legacy tutorials - that should be consulted for existing SQLAlchemy codebases. - - * :doc:`orm/tutorial` - - * :doc:`core/tutorial` - - .. container:: left_right_container .. container:: leftmost diff --git a/doc/build/intro.rst b/doc/build/intro.rst index 01e33df034..686d5f20f5 100644 --- a/doc/build/intro.rst +++ b/doc/build/intro.rst @@ -96,18 +96,13 @@ Installation Guide Supported Platforms ------------------- -SQLAlchemy has been tested against the following platforms: +SQLAlchemy supports the following platforms: -* cPython 2.7 -* cPython 3.6 and higher -* `PyPy `_ 2.1 or greater +* cPython 3.7 and higher +* Python-3 compatible versions of `PyPy `_ -.. versionchanged:: 1.4 - Within the Python 3 series, 3.6 is now the minimum Python 3 version supported. - - .. seealso:: - - :ref:`change_5634` +.. versionchanged:: 2.0 + SQLAlchemy now targets Python 3.7 and above. AsyncIO Support ---------------- @@ -128,9 +123,6 @@ by referring to ``setup.py`` directly or by using `pip `_ or other setuptools-compatible approaches. -.. versionchanged:: 1.1 setuptools is now required by the setup.py file; - plain distutils installs are no longer supported. - Install via pip --------------- @@ -142,7 +134,7 @@ downloaded from PyPI and installed in one step:: This command will download the latest **released** version of SQLAlchemy from the `Python Cheese Shop `_ and install it to your system. -In order to install the latest **prerelease** version, such as ``1.4.0b1``, +In order to install the latest **prerelease** version, such as ``2.0.0b1``, pip requires that the ``--pre`` flag be used:: pip install --pre SQLAlchemy @@ -196,7 +188,7 @@ the available DBAPIs for each database, including external links. Checking the Installed SQLAlchemy Version ------------------------------------------ -This documentation covers SQLAlchemy version 1.4. If you're working on a +This documentation covers SQLAlchemy version 2.0. If you're working on a system that already has SQLAlchemy installed, check the version from your Python prompt like this: @@ -204,11 +196,11 @@ Python prompt like this: >>> import sqlalchemy >>> sqlalchemy.__version__ # doctest: +SKIP - 1.4.0 + 2.0.0 .. _migration: -1.3 to 1.4 Migration +1.x to 2.0 Migration ===================== -Notes on what's changed from 1.3 to 1.4 is available here at :doc:`changelog/migration_14`. +Notes on the new API released in SQLAlchemy 2.0 is available here at :doc:`changelog/migration_20`. diff --git a/doc/build/orm/index.rst b/doc/build/orm/index.rst index 8434df62c7..ed50cc5b31 100644 --- a/doc/build/orm/index.rst +++ b/doc/build/orm/index.rst @@ -11,7 +11,6 @@ tutorial. .. toctree:: :maxdepth: 2 - tutorial mapper_config relationships loading_objects diff --git a/doc/build/orm/tutorial.rst b/doc/build/orm/tutorial.rst deleted file mode 100644 index fb52023420..0000000000 --- a/doc/build/orm/tutorial.rst +++ /dev/null @@ -1,2257 +0,0 @@ -.. _ormtutorial_toplevel: - -==================================== -Object Relational Tutorial (1.x API) -==================================== - -.. admonition:: About this document - - This tutorial covers the well known SQLAlchemy ORM API - that has been in use for many years. As of SQLAlchemy 1.4, there are two - distinct styles of ORM use known as :term:`1.x style` and :term:`2.0 - style`, the latter of which makes a wide range of changes most prominently - around how ORM queries are constructed and executed. - - The plan is that in SQLAlchemy 2.0, the 1.x style of ORM use will be - considered legacy and no longer featured in documentation and many - aspects of it will be removed. However, the most central element of - :term:`1.x style` ORM use, the :class:`_orm.Query` object, will still - remain available for long-term legacy use cases. - - This tutorial is applicable to users who want to learn how SQLAlchemy has - been used for many years, particularly those users working with existing - applications or related learning material that is in 1.x style. - - For an introduction to SQLAlchemy from the new 1.4/2.0 perspective, - see :ref:`unified_tutorial`. - - .. seealso:: - - :ref:`change_5159` - - :ref:`migration_20_toplevel` - - :ref:`unified_tutorial` - -The SQLAlchemy Object Relational Mapper presents a method of associating -user-defined Python classes with database tables, and instances of those -classes (objects) with rows in their corresponding tables. It includes a -system that transparently synchronizes all changes in state between objects -and their related rows, called a :term:`unit of work`, as well as a system -for expressing database queries in terms of the user defined classes and their -defined relationships between each other. - -The ORM is in contrast to the SQLAlchemy Expression Language, upon which the -ORM is constructed. Whereas the SQL Expression Language, introduced in -:ref:`sqlexpression_toplevel`, presents a system of representing the primitive -constructs of the relational database directly without opinion, the ORM -presents a high level and abstracted pattern of usage, which itself is an -example of applied usage of the Expression Language. - -While there is overlap among the usage patterns of the ORM and the Expression -Language, the similarities are more superficial than they may at first appear. -One approaches the structure and content of data from the perspective of a -user-defined :term:`domain model` which is transparently -persisted and refreshed from its underlying storage model. The other -approaches it from the perspective of literal schema and SQL expression -representations which are explicitly composed into messages consumed -individually by the database. - -A successful application may be constructed using the Object Relational Mapper -exclusively. In advanced situations, an application constructed with the ORM -may make occasional usage of the Expression Language directly in certain areas -where specific database interactions are required. - -The following tutorial is in doctest format, meaning each ``>>>`` line -represents something you can type at a Python command prompt, and the -following text represents the expected return value. - -Version Check -============= - -A quick check to verify that we are on at least **version 1.4** of SQLAlchemy:: - - >>> import sqlalchemy - >>> sqlalchemy.__version__ # doctest:+SKIP - 1.4.0 - -Connecting -========== - -For this tutorial we will use an in-memory-only SQLite database. To connect we -use :func:`~sqlalchemy.create_engine`:: - - >>> from sqlalchemy import create_engine - >>> engine = create_engine('sqlite:///:memory:', echo=True) - -The ``echo`` flag is a shortcut to setting up SQLAlchemy logging, which is -accomplished via Python's standard ``logging`` module. With it enabled, we'll -see all the generated SQL produced. If you are working through this tutorial -and want less output generated, set it to ``False``. This tutorial will format -the SQL behind a popup window so it doesn't get in our way; just click the -"SQL" links to see what's being generated. - -The return value of :func:`_sa.create_engine` is an instance of -:class:`_engine.Engine`, and it represents the core interface to the -database, adapted through a :term:`dialect` that handles the details -of the database and :term:`DBAPI` in use. In this case the SQLite -dialect will interpret instructions to the Python built-in ``sqlite3`` -module. - -.. sidebar:: Lazy Connecting - - The :class:`_engine.Engine`, when first returned by :func:`_sa.create_engine`, - has not actually tried to connect to the database yet; that happens - only the first time it is asked to perform a task against the database. - -The first time a method like :meth:`_engine.Engine.execute` or :meth:`_engine.Engine.connect` -is called, the :class:`_engine.Engine` establishes a real :term:`DBAPI` connection to the -database, which is then used to emit the SQL. When using the ORM, we typically -don't use the :class:`_engine.Engine` directly once created; instead, it's used -behind the scenes by the ORM as we'll see shortly. - -.. seealso:: - - :ref:`database_urls` - includes examples of :func:`_sa.create_engine` - connecting to several kinds of databases with links to more information. - -Declare a Mapping -================= - -When using the ORM, the configurational process starts by describing the database -tables we'll be dealing with, and then by defining our own classes which will -be mapped to those tables. In modern SQLAlchemy, -these two tasks are usually performed together, -using a system known as :ref:`declarative_toplevel`, which allows us to create -classes that include directives to describe the actual database table they will -be mapped to. - -Classes mapped using the Declarative system are defined in terms of a base class which -maintains a catalog of classes and -tables relative to that base - this is known as the **declarative base class**. Our -application will usually have just one instance of this base in a commonly -imported module. We create the base class using the :func:`.declarative_base` -function, as follows:: - - >>> from sqlalchemy.orm import declarative_base - - >>> Base = declarative_base() - -Now that we have a "base", we can define any number of mapped classes in terms -of it. We will start with just a single table called ``users``, which will store -records for the end-users using our application. -A new class called ``User`` will be the class to which we map this table. Within -the class, we define details about the table to which we'll be mapping, primarily -the table name, and names and datatypes of columns:: - - >>> from sqlalchemy import Column, Integer, String - >>> class User(Base): - ... __tablename__ = 'users' - ... - ... id = Column(Integer, primary_key=True) - ... name = Column(String) - ... fullname = Column(String) - ... nickname = Column(String) - ... - ... def __repr__(self): - ... return "" % ( - ... self.name, self.fullname, self.nickname) - -.. sidebar:: Tip - - The ``User`` class defines a ``__repr__()`` method, - but note that is **optional**; we only implement it in - this tutorial so that our examples show nicely - formatted ``User`` objects. - -A class using Declarative at a minimum -needs a ``__tablename__`` attribute, and at least one -:class:`_schema.Column` which is part of a primary key [#]_. SQLAlchemy never makes any -assumptions by itself about the table to which -a class refers, including that it has no built-in conventions for names, -datatypes, or constraints. But this doesn't mean -boilerplate is required; instead, you're encouraged to create your -own automated conventions using helper functions and mixin classes, which -is described in detail at :ref:`declarative_mixins`. - -When our class is constructed, Declarative replaces all the :class:`_schema.Column` -objects with special Python accessors known as :term:`descriptors`; this is a -process known as :term:`instrumentation`. The "instrumented" mapped class -will provide us with the means to refer to our table in a SQL context as well -as to persist and load the values of columns from the database. - -Outside of what the mapping process does to our class, the class remains -otherwise mostly a normal Python class, to which we can define any -number of ordinary attributes and methods needed by our application. - -.. [#] For information on why a primary key is required, see - :ref:`faq_mapper_primary_key`. - - -Create a Schema -=============== - -With our ``User`` class constructed via the Declarative system, we have defined information about -our table, known as :term:`table metadata`. The object used by SQLAlchemy to represent -this information for a specific table is called the :class:`_schema.Table` object, and here Declarative has made -one for us. We can see this object by inspecting the ``__table__`` attribute:: - - >>> User.__table__ # doctest: +NORMALIZE_WHITESPACE - Table('users', MetaData(), - Column('id', Integer(), table=, primary_key=True, nullable=False), - Column('name', String(), table=), - Column('fullname', String(), table=), - Column('nickname', String(), table=), schema=None) - -.. sidebar:: Classical Mappings - - The Declarative system, though highly recommended, - is not required in order to use SQLAlchemy's ORM. - Outside of Declarative, any - plain Python class can be mapped to any :class:`_schema.Table` - using the :func:`.mapper` function directly; this - less common usage is described at :ref:`classical_mapping`. - -When we declared our class, Declarative used a Python metaclass in order to -perform additional activities once the class declaration was complete; within -this phase, it then created a :class:`_schema.Table` object according to our -specifications, and associated it with the class by constructing -a :class:`_orm.Mapper` object. This object is a behind-the-scenes object we normally -don't need to deal with directly (though it can provide plenty of information -about our mapping when we need it). - -The :class:`_schema.Table` object is a member of a larger collection -known as :class:`_schema.MetaData`. When using Declarative, -this object is available using the ``.metadata`` -attribute of our declarative base class. - -The :class:`_schema.MetaData` -is a :term:`registry` which includes the ability to emit a limited set -of schema generation commands to the database. As our SQLite database -does not actually have a ``users`` table present, we can use :class:`_schema.MetaData` -to issue CREATE TABLE statements to the database for all tables that don't yet exist. -Below, we call the :meth:`_schema.MetaData.create_all` method, passing in our :class:`_engine.Engine` -as a source of database connectivity. We will see that special commands are -first emitted to check for the presence of the ``users`` table, and following that -the actual ``CREATE TABLE`` statement: - -.. sourcecode:: python+sql - - >>> Base.metadata.create_all(engine) - BEGIN... - CREATE TABLE users ( - id INTEGER NOT NULL, - name VARCHAR, - fullname VARCHAR, - nickname VARCHAR, - PRIMARY KEY (id) - ) - [...] () - COMMIT - -.. topic:: Minimal Table Descriptions vs. Full Descriptions - - Users familiar with the syntax of CREATE TABLE may notice that the - VARCHAR columns were generated without a length; on SQLite and PostgreSQL, - this is a valid datatype, but on others, it's not allowed. So if running - this tutorial on one of those databases, and you wish to use SQLAlchemy to - issue CREATE TABLE, a "length" may be provided to the :class:`~sqlalchemy.types.String` type as - below:: - - Column(String(50)) - - The length field on :class:`~sqlalchemy.types.String`, as well as similar precision/scale fields - available on :class:`~sqlalchemy.types.Integer`, :class:`~sqlalchemy.types.Numeric`, etc. are not referenced by - SQLAlchemy other than when creating tables. - - Additionally, Firebird and Oracle require sequences to generate new - primary key identifiers, and SQLAlchemy doesn't generate or assume these - without being instructed. For that, you use the :class:`~sqlalchemy.schema.Sequence` construct:: - - from sqlalchemy import Sequence - Column(Integer, Sequence('user_id_seq'), primary_key=True) - - A full, foolproof :class:`~sqlalchemy.schema.Table` generated via our declarative - mapping is therefore:: - - class User(Base): - __tablename__ = 'users' - id = Column(Integer, Sequence('user_id_seq'), primary_key=True) - name = Column(String(50)) - fullname = Column(String(50)) - nickname = Column(String(50)) - - def __repr__(self): - return "" % ( - self.name, self.fullname, self.nickname) - - We include this more verbose table definition separately - to highlight the difference between a minimal construct geared primarily - towards in-Python usage only, versus one that will be used to emit CREATE - TABLE statements on a particular set of backends with more stringent - requirements. - -Create an Instance of the Mapped Class -====================================== - -With mappings complete, let's now create and inspect a ``User`` object:: - - >>> ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname') - >>> ed_user.name - 'ed' - >>> ed_user.nickname - 'edsnickname' - >>> str(ed_user.id) - 'None' - - -.. sidebar:: the ``__init__()`` method - - Our ``User`` class, as defined using the Declarative system, has - been provided with a constructor (e.g. ``__init__()`` method) which automatically - accepts keyword names that match the columns we've mapped. We are free - to define any explicit ``__init__()`` method we prefer on our class, which - will override the default method provided by Declarative. - -Even though we didn't specify it in the constructor, the ``id`` attribute -still produces a value of ``None`` when we access it (as opposed to Python's -usual behavior of raising ``AttributeError`` for an undefined attribute). -SQLAlchemy's :term:`instrumentation` normally produces this default value for -column-mapped attributes when first accessed. For those attributes where -we've actually assigned a value, the instrumentation system is tracking -those assignments for use within an eventual INSERT statement to be emitted to the -database. - -Creating a Session -================== - -We're now ready to start talking to the database. The ORM's "handle" to the -database is the :class:`~sqlalchemy.orm.session.Session`. When we first set up -the application, at the same level as our :func:`~sqlalchemy.create_engine` -statement, we define a :class:`~sqlalchemy.orm.session.Session` class which -will serve as a factory for new :class:`~sqlalchemy.orm.session.Session` -objects:: - - >>> from sqlalchemy.orm import sessionmaker - >>> Session = sessionmaker(bind=engine) - -In the case where your application does not yet have an -:class:`~sqlalchemy.engine.Engine` when you define your module-level -objects, just set it up like this:: - - >>> Session = sessionmaker() - -Later, when you create your engine with :func:`~sqlalchemy.create_engine`, -connect it to the :class:`~sqlalchemy.orm.session.Session` using -:meth:`~.sessionmaker.configure`:: - - >>> Session.configure(bind=engine) # once engine is available - -.. Setup code, not for display - ensure no cascade_backrefs warnings occur - - >>> Session.configure(future=True) - -.. sidebar:: Session Lifecycle Patterns - - The question of when to make a :class:`.Session` depends a lot on what - kind of application is being built. Keep in mind, - the :class:`.Session` is just a workspace for your objects, - local to a particular database connection - if you think of - an application thread as a guest at a dinner party, the :class:`.Session` - is the guest's plate and the objects it holds are the food - (and the database...the kitchen?)! More on this topic - available at :ref:`session_faq_whentocreate`. - -This custom-made :class:`~sqlalchemy.orm.session.Session` class will create -new :class:`~sqlalchemy.orm.session.Session` objects which are bound to our -database. Other transactional characteristics may be defined when calling -:class:`~.sessionmaker` as well; these are described in a later -chapter. Then, whenever you need to have a conversation with the database, you -instantiate a :class:`~sqlalchemy.orm.session.Session`:: - - >>> session = Session() - -The above :class:`~sqlalchemy.orm.session.Session` is associated with our -SQLite-enabled :class:`_engine.Engine`, but it hasn't opened any connections yet. When it's first -used, it retrieves a connection from a pool of connections maintained by the -:class:`_engine.Engine`, and holds onto it until we commit all changes and/or close the -session object. - - -Adding and Updating Objects -=========================== - -To persist our ``User`` object, we :meth:`~.Session.add` it to our :class:`~sqlalchemy.orm.session.Session`:: - - >>> ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname') - >>> session.add(ed_user) - -At this point, we say that the instance is **pending**; no SQL has yet been issued -and the object is not yet represented by a row in the database. The -:class:`~sqlalchemy.orm.session.Session` will issue the SQL to persist ``Ed -Jones`` as soon as is needed, using a process known as a **flush**. If we -query the database for ``Ed Jones``, all pending information will first be -flushed, and the query is issued immediately thereafter. - -For example, below we create a new :class:`~sqlalchemy.orm.query.Query` object -which loads instances of ``User``. We "filter by" the ``name`` attribute of -``ed``, and indicate that we'd like only the first result in the full list of -rows. A ``User`` instance is returned which is equivalent to that which we've -added: - -.. sourcecode:: python+sql - - {sql}>>> our_user = session.query(User).filter_by(name='ed').first() # doctest:+NORMALIZE_WHITESPACE - BEGIN (implicit) - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('ed', 'Ed Jones', 'edsnickname') - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ? - LIMIT ? OFFSET ? - [...] ('ed', 1, 0) - {stop}>>> our_user - - -In fact, the :class:`~sqlalchemy.orm.session.Session` has identified that the -row returned is the **same** row as one already represented within its -internal map of objects, so we actually got back the identical instance as -that which we just added:: - - >>> ed_user is our_user - True - -The ORM concept at work here is known as an :term:`identity map` -and ensures that -all operations upon a particular row within a -:class:`~sqlalchemy.orm.session.Session` operate upon the same set of data. -Once an object with a particular primary key is present in the -:class:`~sqlalchemy.orm.session.Session`, all SQL queries on that -:class:`~sqlalchemy.orm.session.Session` will always return the same Python -object for that particular primary key; it also will raise an error if an -attempt is made to place a second, already-persisted object with the same -primary key within the session. - -We can add more ``User`` objects at once using -:func:`~sqlalchemy.orm.session.Session.add_all`: - -.. sourcecode:: python+sql - - >>> session.add_all([ - ... User(name='wendy', fullname='Wendy Williams', nickname='windy'), - ... User(name='mary', fullname='Mary Contrary', nickname='mary'), - ... User(name='fred', fullname='Fred Flintstone', nickname='freddy')]) - -Also, we've decided Ed's nickname isn't that great, so lets change it: - -.. sourcecode:: python+sql - - >>> ed_user.nickname = 'eddie' - -The :class:`~sqlalchemy.orm.session.Session` is paying attention. It knows, -for example, that ``Ed Jones`` has been modified: - -.. sourcecode:: python+sql - - >>> session.dirty - IdentitySet([]) - -and that three new ``User`` objects are pending: - -.. sourcecode:: python+sql - - >>> session.new # doctest: +SKIP - IdentitySet([, - , - ]) - -We tell the :class:`~sqlalchemy.orm.session.Session` that we'd like to issue -all remaining changes to the database and commit the transaction, which has -been in progress throughout. We do this via :meth:`~.Session.commit`. The -:class:`~sqlalchemy.orm.session.Session` emits the ``UPDATE`` statement -for the nickname change on "ed", as well as ``INSERT`` statements for the -three new ``User`` objects we've added: - -.. sourcecode:: python+sql - - {sql}>>> session.commit() - UPDATE users SET nickname=? WHERE users.id = ? - [...] ('eddie', 1) - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('wendy', 'Wendy Williams', 'windy') - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('mary', 'Mary Contrary', 'mary') - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('fred', 'Fred Flintstone', 'freddy') - COMMIT - -:meth:`~.Session.commit` flushes the remaining changes to the -database, and commits the transaction. The connection resources referenced by -the session are now returned to the connection pool. Subsequent operations -with this session will occur in a **new** transaction, which will again -re-acquire connection resources when first needed. - -If we look at Ed's ``id`` attribute, which earlier was ``None``, it now has a value: - -.. sourcecode:: python+sql - - {sql}>>> ed_user.id # doctest: +NORMALIZE_WHITESPACE - BEGIN (implicit) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.id = ? - [...] (1,) - {stop}1 - -After the :class:`~sqlalchemy.orm.session.Session` inserts new rows in the -database, all newly generated identifiers and database-generated defaults -become available on the instance, either immediately or via -load-on-first-access. In this case, the entire row was re-loaded on access -because a new transaction was begun after we issued :meth:`~.Session.commit`. SQLAlchemy -by default refreshes data from a previous transaction the first time it's -accessed within a new transaction, so that the most recent state is available. -The level of reloading is configurable as is described in :doc:`/orm/session`. - -.. topic:: Session Object States - - As our ``User`` object moved from being outside the :class:`.Session`, to - inside the :class:`.Session` without a primary key, to actually being - inserted, it moved between three out of five - available "object states" - **transient**, **pending**, and **persistent**. - Being aware of these states and what they mean is always a good idea - - be sure to read :ref:`session_object_states` for a quick overview. - -Rolling Back -============ -Since the :class:`~sqlalchemy.orm.session.Session` works within a transaction, -we can roll back changes made too. Let's make two changes that we'll revert; -``ed_user``'s user name gets set to ``Edwardo``: - -.. sourcecode:: python+sql - - >>> ed_user.name = 'Edwardo' - -and we'll add another erroneous user, ``fake_user``: - -.. sourcecode:: python+sql - - >>> fake_user = User(name='fakeuser', fullname='Invalid', nickname='12345') - >>> session.add(fake_user) - -Querying the session, we can see that they're flushed into the current transaction: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).filter(User.name.in_(['Edwardo', 'fakeuser'])).all() - UPDATE users SET name=? WHERE users.id = ? - [...] ('Edwardo', 1) - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('fakeuser', 'Invalid', '12345') - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name IN (?, ?) - [...] ('Edwardo', 'fakeuser') - {stop}[, ] - -Rolling back, we can see that ``ed_user``'s name is back to ``ed``, and -``fake_user`` has been kicked out of the session: - -.. sourcecode:: python+sql - - {sql}>>> session.rollback() - ROLLBACK - {stop} - - {sql}>>> ed_user.name - BEGIN (implicit) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.id = ? - [...] (1,) - {stop}u'ed' - >>> fake_user in session - False - -issuing a SELECT illustrates the changes made to the database: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).filter(User.name.in_(['ed', 'fakeuser'])).all() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name IN (?, ?) - [...] ('ed', 'fakeuser') - {stop}[] - -.. _ormtutorial_querying: - -Querying -======== - -A :class:`~sqlalchemy.orm.query.Query` object is created using the -:class:`~sqlalchemy.orm.session.Session.query()` method on -:class:`~sqlalchemy.orm.session.Session`. This function takes a variable -number of arguments, which can be any combination of classes and -class-instrumented descriptors. Below, we indicate a -:class:`~sqlalchemy.orm.query.Query` which loads ``User`` instances. When -evaluated in an iterative context, the list of ``User`` objects present is -returned: - -.. sourcecode:: python+sql - - {sql}>>> for instance in session.query(User).order_by(User.id): - ... print(instance.name, instance.fullname) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users ORDER BY users.id - [...] () - {stop}ed Ed Jones - wendy Wendy Williams - mary Mary Contrary - fred Fred Flintstone - -The :class:`~sqlalchemy.orm.query.Query` also accepts ORM-instrumented -descriptors as arguments. Any time multiple class entities or column-based -entities are expressed as arguments to the -:class:`~sqlalchemy.orm.session.Session.query()` function, the return result -is expressed as tuples: - -.. sourcecode:: python+sql - - {sql}>>> for name, fullname in session.query(User.name, User.fullname): - ... print(name, fullname) - SELECT users.name AS users_name, - users.fullname AS users_fullname - FROM users - [...] () - {stop}ed Ed Jones - wendy Wendy Williams - mary Mary Contrary - fred Fred Flintstone - -The tuples returned by :class:`~sqlalchemy.orm.query.Query` are *named* -tuples, supplied by the :class:`.Row` class, and can be treated much like an -ordinary Python object. The names are -the same as the attribute's name for an attribute, and the class name for a -class: - -.. sourcecode:: python+sql - - {sql}>>> for row in session.query(User, User.name).all(): - ... print(row.User, row.name) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - users.name AS users_name__1 - FROM users - [...] () - {stop} ed - wendy - mary - fred - -You can control the names of individual column expressions using the -:meth:`_expression.ColumnElement.label` construct, which is available from -any :class:`_expression.ColumnElement`-derived object, as well as any class attribute which -is mapped to one (such as ``User.name``): - -.. sourcecode:: python+sql - - {sql}>>> for row in session.query(User.name.label('name_label')).all(): - ... print(row.name_label) - SELECT users.name AS name_label - FROM users - [...] (){stop} - ed - wendy - mary - fred - -The name given to a full entity such as ``User``, assuming that multiple -entities are present in the call to :meth:`~.Session.query`, can be controlled using -:func:`~.sqlalchemy.orm.aliased` : - -.. sourcecode:: python+sql - - >>> from sqlalchemy.orm import aliased - >>> user_alias = aliased(User, name='user_alias') - - {sql}>>> for row in session.query(user_alias, user_alias.name).all(): - ... print(row.user_alias) - SELECT user_alias.id AS user_alias_id, - user_alias.name AS user_alias_name, - user_alias.fullname AS user_alias_fullname, - user_alias.nickname AS user_alias_nickname, - user_alias.name AS user_alias_name__1 - FROM users AS user_alias - [...] (){stop} - - - - - -Basic operations with :class:`~sqlalchemy.orm.query.Query` include issuing -LIMIT and OFFSET, most conveniently using Python array slices and typically in -conjunction with ORDER BY: - -.. sourcecode:: python+sql - - {sql}>>> for u in session.query(User).order_by(User.id)[1:3]: - ... print(u) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users ORDER BY users.id - LIMIT ? OFFSET ? - [...] (2, 1){stop} - - - -and filtering results, which is accomplished either with -:func:`~sqlalchemy.orm.query.Query.filter_by`, which uses keyword arguments: - -.. sourcecode:: python+sql - - {sql}>>> for name, in session.query(User.name).\ - ... filter_by(fullname='Ed Jones'): - ... print(name) - SELECT users.name AS users_name FROM users - WHERE users.fullname = ? - [...] ('Ed Jones',) - {stop}ed - -...or :func:`~sqlalchemy.orm.query.Query.filter`, which uses more flexible SQL -expression language constructs. These allow you to use regular Python -operators with the class-level attributes on your mapped class: - -.. sourcecode:: python+sql - - {sql}>>> for name, in session.query(User.name).\ - ... filter(User.fullname=='Ed Jones'): - ... print(name) - SELECT users.name AS users_name FROM users - WHERE users.fullname = ? - [...] ('Ed Jones',) - {stop}ed - -The :class:`~sqlalchemy.orm.query.Query` object is fully **generative**, meaning -that most method calls return a new :class:`~sqlalchemy.orm.query.Query` -object upon which further criteria may be added. For example, to query for -users named "ed" with a full name of "Ed Jones", you can call -:func:`~sqlalchemy.orm.query.Query.filter` twice, which joins criteria using -``AND``: - -.. sourcecode:: python+sql - - {sql}>>> for user in session.query(User).\ - ... filter(User.name=='ed').\ - ... filter(User.fullname=='Ed Jones'): - ... print(user) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ? AND users.fullname = ? - [...] ('ed', 'Ed Jones') - {stop} - -Common Filter Operators ------------------------ - -Here's a rundown of some of the most common operators used in -:func:`~sqlalchemy.orm.query.Query.filter`: - -* :meth:`equals <.ColumnOperators.__eq__>`:: - - query.filter(User.name == 'ed') - -* :meth:`not equals <.ColumnOperators.__ne__>`:: - - query.filter(User.name != 'ed') - -* :meth:`LIKE <.ColumnOperators.like>`:: - - query.filter(User.name.like('%ed%')) - - .. note:: :meth:`.ColumnOperators.like` renders the LIKE operator, which - is case insensitive on some backends, and case sensitive - on others. For guaranteed case-insensitive comparisons, use - :meth:`.ColumnOperators.ilike`. - -* :meth:`ILIKE <.ColumnOperators.ilike>` (case-insensitive LIKE):: - - query.filter(User.name.ilike('%ed%')) - - .. note:: most backends don't support ILIKE directly. For those, - the :meth:`.ColumnOperators.ilike` operator renders an expression - combining LIKE with the LOWER SQL function applied to each operand. - -* :meth:`IN <.ColumnOperators.in_>`:: - - query.filter(User.name.in_(['ed', 'wendy', 'jack'])) - - # works with query objects too: - query.filter(User.name.in_( - session.query(User.name).filter(User.name.like('%ed%')) - )) - - # use tuple_() for composite (multi-column) queries - from sqlalchemy import tuple_ - query.filter( - tuple_(User.name, User.nickname).\ - in_([('ed', 'edsnickname'), ('wendy', 'windy')]) - ) - -* :meth:`NOT IN <.ColumnOperators.not_in>`:: - - query.filter(~User.name.in_(['ed', 'wendy', 'jack'])) - -* :meth:`IS NULL <.ColumnOperators.is_>`:: - - query.filter(User.name == None) - - # alternatively, if pep8/linters are a concern - query.filter(User.name.is_(None)) - -* :meth:`IS NOT NULL <.ColumnOperators.is_not>`:: - - query.filter(User.name != None) - - # alternatively, if pep8/linters are a concern - query.filter(User.name.is_not(None)) - -* :func:`AND <.sql.expression.and_>`:: - - # use and_() - from sqlalchemy import and_ - query.filter(and_(User.name == 'ed', User.fullname == 'Ed Jones')) - - # or send multiple expressions to .filter() - query.filter(User.name == 'ed', User.fullname == 'Ed Jones') - - # or chain multiple filter()/filter_by() calls - query.filter(User.name == 'ed').filter(User.fullname == 'Ed Jones') - - .. note:: Make sure you use :func:`.and_` and **not** the - Python ``and`` operator! - -* :func:`OR <.sql.expression.or_>`:: - - from sqlalchemy import or_ - query.filter(or_(User.name == 'ed', User.name == 'wendy')) - - .. note:: Make sure you use :func:`.or_` and **not** the - Python ``or`` operator! - -* :meth:`MATCH <.ColumnOperators.match>`:: - - query.filter(User.name.match('wendy')) - - .. note:: - - :meth:`~.ColumnOperators.match` uses a database-specific ``MATCH`` - or ``CONTAINS`` function; its behavior will vary by backend and is not - available on some backends such as SQLite. - -.. _orm_tutorial_query_returning: - -Returning Lists and Scalars ---------------------------- - -A number of methods on :class:`_query.Query` -immediately issue SQL and return a value containing loaded -database results. Here's a brief tour: - -* :meth:`_query.Query.all` returns a list: - - .. sourcecode:: python+sql - - >>> query = session.query(User).filter(User.name.like('%ed')).order_by(User.id) - {sql}>>> query.all() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name LIKE ? ORDER BY users.id - [...] ('%ed',) - {stop}[, - ] - - .. warning:: - - When the :class:`_query.Query` object returns lists of ORM-mapped objects - such as the ``User`` object above, the entries are **deduplicated** - based on primary key, as the results are interpreted from the SQL - result set. That is, if SQL query returns a row with ``id=7`` twice, - you would only get a single ``User(id=7)`` object back in the result - list. This does not apply to the case when individual columns are - queried. - - .. seealso:: - - :ref:`faq_query_deduplicating` - - -* :meth:`_query.Query.first` applies a limit of one and returns - the first result as a scalar: - - .. sourcecode:: python+sql - - {sql}>>> query.first() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name LIKE ? ORDER BY users.id - LIMIT ? OFFSET ? - [...] ('%ed', 1, 0) - {stop} - -* :meth:`_query.Query.one` fully fetches all rows, and if not - exactly one object identity or composite row is present in the result, raises - an error. With multiple rows found: - - .. sourcecode:: python+sql - - >>> user = query.one() - Traceback (most recent call last): - ... - MultipleResultsFound: Multiple rows were found for one() - - With no rows found: - - .. sourcecode:: python+sql - - >>> user = query.filter(User.id == 99).one() - Traceback (most recent call last): - ... - NoResultFound: No row was found for one() - - The :meth:`_query.Query.one` method is great for systems that expect to handle - "no items found" versus "multiple items found" differently; such as a RESTful - web service, which may want to raise a "404 not found" when no results are found, - but raise an application error when multiple results are found. - -* :meth:`_query.Query.one_or_none` is like :meth:`_query.Query.one`, except that if no - results are found, it doesn't raise an error; it just returns ``None``. Like - :meth:`_query.Query.one`, however, it does raise an error if multiple results are - found. - -* :meth:`_query.Query.scalar` invokes the :meth:`_query.Query.one` method, and upon - success returns the first column of the row: - - .. sourcecode:: python+sql - - >>> query = session.query(User.id).filter(User.name == 'ed').\ - ... order_by(User.id) - {sql}>>> query.scalar() - SELECT users.id AS users_id - FROM users - WHERE users.name = ? ORDER BY users.id - [...] ('ed',) - {stop}1 - -.. _orm_tutorial_literal_sql: - -Using Textual SQL ------------------ - -Literal strings can be used flexibly with -:class:`~sqlalchemy.orm.query.Query`, by specifying their use -with the :func:`_expression.text` construct, which is accepted -by most applicable methods. For example, -:meth:`_query.Query.filter` and -:meth:`_query.Query.order_by`: - -.. sourcecode:: python+sql - - >>> from sqlalchemy import text - {sql}>>> for user in session.query(User).\ - ... filter(text("id<224")).\ - ... order_by(text("id")).all(): - ... print(user.name) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE id<224 ORDER BY id - [...] () - {stop}ed - wendy - mary - fred - -Bind parameters can be specified with string-based SQL, using a colon. To -specify the values, use the :meth:`_query.Query.params` -method: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).filter(text("id<:value and name=:name")).\ - ... params(value=224, name='fred').order_by(User.id).one() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE id - -To use an entirely string-based statement, a :func:`_expression.text` construct -representing a complete statement can be passed to -:meth:`_query.Query.from_statement`. Without further -specification, the ORM will match columns in the ORM mapping to the result -returned by the SQL statement based on column name: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).from_statement( - ... text("SELECT * FROM users where name=:name")).params(name='ed').all() - SELECT * FROM users where name=? - [...] ('ed',) - {stop}[] - -For better targeting of mapped columns to a textual SELECT, as well as to -match on a specific subset of columns in arbitrary order, individual mapped -columns are passed in the desired order to :meth:`_expression.TextClause.columns`: - -.. sourcecode:: python+sql - - >>> stmt = text("SELECT name, id, fullname, nickname " - ... "FROM users where name=:name") - >>> stmt = stmt.columns(User.name, User.id, User.fullname, User.nickname) - {sql}>>> session.query(User).from_statement(stmt).params(name='ed').all() - SELECT name, id, fullname, nickname FROM users where name=? - [...] ('ed',) - {stop}[] - -When selecting from a :func:`_expression.text` construct, the :class:`_query.Query` -may still specify what columns and entities are to be returned; instead of -``query(User)`` we can also ask for the columns individually, as in -any other case: - -.. sourcecode:: python+sql - - >>> stmt = text("SELECT name, id FROM users where name=:name") - >>> stmt = stmt.columns(User.name, User.id) - {sql}>>> session.query(User.id, User.name).\ - ... from_statement(stmt).params(name='ed').all() - SELECT name, id FROM users where name=? - [...] ('ed',) - {stop}[(1, u'ed')] - -.. seealso:: - - :ref:`sqlexpression_text` - The :func:`_expression.text` construct explained - from the perspective of Core-only queries. - -Counting --------- - -:class:`~sqlalchemy.orm.query.Query` includes a convenience method for -counting called :meth:`_query.Query.count`: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).filter(User.name.like('%ed')).count() - SELECT count(*) AS count_1 - FROM (SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name LIKE ?) AS anon_1 - [...] ('%ed',) - {stop}2 - -.. sidebar:: Counting on ``count()`` - - :meth:`_query.Query.count` used to be a very complicated method - when it would try to guess whether or not a subquery was needed - around the - existing query, and in some exotic cases it wouldn't do the right thing. - Now that it uses a simple subquery every time, it's only two lines long - and always returns the right answer. Use ``func.count()`` if a - particular statement absolutely cannot tolerate the subquery being present. - -The :meth:`_query.Query.count` method is used to determine -how many rows the SQL statement would return. Looking -at the generated SQL above, SQLAlchemy always places whatever it is we are -querying into a subquery, then counts the rows from that. In some cases -this can be reduced to a simpler ``SELECT count(*) FROM table``, however -modern versions of SQLAlchemy don't try to guess when this is appropriate, -as the exact SQL can be emitted using more explicit means. - -For situations where the "thing to be counted" needs -to be indicated specifically, we can specify the "count" function -directly using the expression ``func.count()``, available from the -:attr:`~sqlalchemy.sql.expression.func` construct. Below we -use it to return the count of each distinct user name: - -.. sourcecode:: python+sql - - >>> from sqlalchemy import func - {sql}>>> session.query(func.count(User.name), User.name).group_by(User.name).all() - SELECT count(users.name) AS count_1, users.name AS users_name - FROM users GROUP BY users.name - [...] () - {stop}[(1, u'ed'), (1, u'fred'), (1, u'mary'), (1, u'wendy')] - -To achieve our simple ``SELECT count(*) FROM table``, we can apply it as: - -.. sourcecode:: python+sql - - {sql}>>> session.query(func.count('*')).select_from(User).scalar() - SELECT count(?) AS count_1 - FROM users - [...] ('*',) - {stop}4 - -The usage of :meth:`_query.Query.select_from` can be removed if we express the count in terms -of the ``User`` primary key directly: - -.. sourcecode:: python+sql - - {sql}>>> session.query(func.count(User.id)).scalar() - SELECT count(users.id) AS count_1 - FROM users - [...] () - {stop}4 - -.. _orm_tutorial_relationship: - -Building a Relationship -======================= - -Let's consider how a second table, related to ``User``, can be mapped and -queried. Users in our system -can store any number of email addresses associated with their username. This -implies a basic one to many association from the ``users`` to a new -table which stores email addresses, which we will call ``addresses``. Using -declarative, we define this table along with its mapped class, ``Address``: - -.. sourcecode:: python - - >>> from sqlalchemy import ForeignKey - >>> from sqlalchemy.orm import relationship - - >>> class Address(Base): - ... __tablename__ = 'addresses' - ... id = Column(Integer, primary_key=True) - ... email_address = Column(String, nullable=False) - ... user_id = Column(Integer, ForeignKey('users.id')) - ... - ... user = relationship("User", back_populates="addresses") - ... - ... def __repr__(self): - ... return "" % self.email_address - - >>> User.addresses = relationship( - ... "Address", order_by=Address.id, back_populates="user") - -The above class introduces the :class:`_schema.ForeignKey` construct, which is a -directive applied to :class:`_schema.Column` that indicates that values in this -column should be :term:`constrained` to be values present in the named remote -column. This is a core feature of relational databases, and is the "glue" that -transforms an otherwise unconnected collection of tables to have rich -overlapping relationships. The :class:`_schema.ForeignKey` above expresses that -values in the ``addresses.user_id`` column should be constrained to -those values in the ``users.id`` column, i.e. its primary key. - -A second directive, known as :func:`_orm.relationship`, -tells the ORM that the ``Address`` class itself should be linked -to the ``User`` class, using the attribute ``Address.user``. -:func:`_orm.relationship` uses the foreign key -relationships between the two tables to determine the nature of -this linkage, determining that ``Address.user`` will be :term:`many to one`. -An additional :func:`_orm.relationship` directive is placed on the -``User`` mapped class under the attribute ``User.addresses``. In both -:func:`_orm.relationship` directives, the parameter -:paramref:`_orm.relationship.back_populates` is assigned to refer to the -complementary attribute names; by doing so, each :func:`_orm.relationship` -can make intelligent decision about the same relationship as expressed -in reverse; on one side, ``Address.user`` refers to a ``User`` instance, -and on the other side, ``User.addresses`` refers to a list of -``Address`` instances. - -.. note:: - - The :paramref:`_orm.relationship.back_populates` parameter is a newer - version of a very common SQLAlchemy feature called - :paramref:`_orm.relationship.backref`. The :paramref:`_orm.relationship.backref` - parameter hasn't gone anywhere and will always remain available! - The :paramref:`_orm.relationship.back_populates` is the same thing, except - a little more verbose and easier to manipulate. For an overview - of the entire topic, see the section :ref:`relationships_backref`. - -The reverse side of a many-to-one relationship is always :term:`one to many`. -A full catalog of available :func:`_orm.relationship` configurations -is at :ref:`relationship_patterns`. - -The two complementing relationships ``Address.user`` and ``User.addresses`` -are referred to as a :term:`bidirectional relationship`, and is a key -feature of the SQLAlchemy ORM. The section :ref:`relationships_backref` -discusses the "backref" feature in detail. - -Arguments to :func:`_orm.relationship` which concern the remote class -can be specified using strings, assuming the Declarative system is in -use. Once all mappings are complete, these strings are evaluated -as Python expressions in order to produce the actual argument, in the -above case the ``User`` class. The names which are allowed during -this evaluation include, among other things, the names of all classes -which have been created in terms of the declared base. - -See the docstring for :func:`_orm.relationship` for more detail on argument style. - -.. topic:: Did you know ? - - * a FOREIGN KEY constraint in most (though not all) relational databases can - only link to a primary key column, or a column that has a UNIQUE constraint. - * a FOREIGN KEY constraint that refers to a multiple column primary key, and itself - has multiple columns, is known as a "composite foreign key". It can also - reference a subset of those columns. - * FOREIGN KEY columns can automatically update themselves, in response to a change - in the referenced column or row. This is known as the CASCADE *referential action*, - and is a built in function of the relational database. - * FOREIGN KEY can refer to its own table. This is referred to as a "self-referential" - foreign key. - * Read more about foreign keys at `Foreign Key - Wikipedia `_. - -We'll need to create the ``addresses`` table in the database, so we will issue -another CREATE from our metadata, which will skip over tables which have -already been created: - -.. sourcecode:: python+sql - - {sql}>>> Base.metadata.create_all(engine) - BEGIN... - CREATE TABLE addresses ( - id INTEGER NOT NULL, - email_address VARCHAR NOT NULL, - user_id INTEGER, - PRIMARY KEY (id), - FOREIGN KEY(user_id) REFERENCES users (id) - ) - [...] () - COMMIT - -Working with Related Objects -============================ - -Now when we create a ``User``, a blank ``addresses`` collection will be -present. Various collection types, such as sets and dictionaries, are possible -here (see :ref:`custom_collections` for details), but by -default, the collection is a Python list. - -.. sourcecode:: python+sql - - >>> jack = User(name='jack', fullname='Jack Bean', nickname='gjffdd') - >>> jack.addresses - [] - -We are free to add ``Address`` objects on our ``User`` object. In this case we -just assign a full list directly: - -.. sourcecode:: python+sql - - >>> jack.addresses = [ - ... Address(email_address='jack@google.com'), - ... Address(email_address='j25@yahoo.com')] - -When using a bidirectional relationship, elements added in one direction -automatically become visible in the other direction. This behavior occurs -based on attribute on-change events and is evaluated in Python, without -using any SQL: - -.. sourcecode:: python+sql - - >>> jack.addresses[1] - - - >>> jack.addresses[1].user - - -Let's add and commit ``Jack Bean`` to the database. ``jack`` as well -as the two ``Address`` members in the corresponding ``addresses`` -collection are both added to the session at once, using a process -known as **cascading**: - -.. sourcecode:: python+sql - - >>> session.add(jack) - {sql}>>> session.commit() - INSERT INTO users (name, fullname, nickname) VALUES (?, ?, ?) - [...] ('jack', 'Jack Bean', 'gjffdd') - INSERT INTO addresses (email_address, user_id) VALUES (?, ?) - [...] ('jack@google.com', 5) - INSERT INTO addresses (email_address, user_id) VALUES (?, ?) - [...] ('j25@yahoo.com', 5) - COMMIT - -Querying for Jack, we get just Jack back. No SQL is yet issued for Jack's addresses: - -.. sourcecode:: python+sql - - {sql}>>> jack = session.query(User).\ - ... filter_by(name='jack').one() - BEGIN (implicit) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ? - [...] ('jack',) - - {stop}>>> jack - - -Let's look at the ``addresses`` collection. Watch the SQL: - -.. sourcecode:: python+sql - - {sql}>>> jack.addresses - SELECT addresses.id AS addresses_id, - addresses.email_address AS - addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE ? = addresses.user_id ORDER BY addresses.id - [...] (5,) - {stop}[, ] - -When we accessed the ``addresses`` collection, SQL was suddenly issued. This -is an example of a :term:`lazy loading` relationship. The ``addresses`` collection -is now loaded and behaves just like an ordinary list. We'll cover ways -to optimize the loading of this collection in a bit. - -.. _ormtutorial_joins: - -Querying with Joins -=================== - -Now that we have two tables, we can show some more features of :class:`_query.Query`, -specifically how to create queries that deal with both tables at the same time. -The `Wikipedia page on SQL JOIN -`_ offers a good introduction to -join techniques, several of which we'll illustrate here. - -To construct a simple implicit join between ``User`` and ``Address``, -we can use :meth:`_query.Query.filter` to equate their related columns together. -Below we load the ``User`` and ``Address`` entities at once using this method: - -.. sourcecode:: python+sql - - {sql}>>> for u, a in session.query(User, Address).\ - ... filter(User.id==Address.user_id).\ - ... filter(Address.email_address=='jack@google.com').\ - ... all(): - ... print(u) - ... print(a) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM users, addresses - WHERE users.id = addresses.user_id - AND addresses.email_address = ? - [...] ('jack@google.com',) - {stop} - - -The actual SQL JOIN syntax, on the other hand, is most easily achieved -using the :meth:`_query.Query.join` method: - -.. sourcecode:: python+sql - - {sql}>>> session.query(User).join(Address).\ - ... filter(Address.email_address=='jack@google.com').\ - ... all() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users JOIN addresses ON users.id = addresses.user_id - WHERE addresses.email_address = ? - [...] ('jack@google.com',) - {stop}[] - -:meth:`_query.Query.join` knows how to join between ``User`` -and ``Address`` because there's only one foreign key between them. If there -were no foreign keys, or several, :meth:`_query.Query.join` -works better when one of the following forms are used:: - - query.join(Address, User.id==Address.user_id) # explicit condition - query.join(User.addresses) # specify relationship from left to right - query.join(Address, User.addresses) # same, with explicit target - query.join(User.addresses.and_(Address.name != 'foo')) # use relationship + additional ON criteria - -As you would expect, the same idea is used for "outer" joins, using the -:meth:`_query.Query.outerjoin` function:: - - query.outerjoin(User.addresses) # LEFT OUTER JOIN - -The reference documentation for :meth:`_query.Query.join` contains detailed information -and examples of the calling styles accepted by this method; :meth:`_query.Query.join` -is an important method at the center of usage for any SQL-fluent application. - -.. topic:: What does :class:`_query.Query` select from if there's multiple entities? - - The :meth:`_query.Query.join` method will **typically join from the leftmost - item** in the list of entities, when the ON clause is omitted, or if the - ON clause is a plain SQL expression. To control the first entity in the list - of JOINs, use the :meth:`_query.Query.select_from` method:: - - query = session.query(User, Address).select_from(Address).join(User) - - -.. _ormtutorial_aliases: - -Using Aliases -------------- - -When querying across multiple tables, if the same table needs to be referenced -more than once, SQL typically requires that the table be *aliased* with -another name, so that it can be distinguished against other occurrences of -that table. This is supported using the -:func:`_orm.aliased` construct. When joining to relationships using -using :func:`_orm.aliased`, the special attribute method -:meth:`_orm.PropComparator.of_type` may be used to alter the target of -a relationship join to refer to a given :func:`_orm.aliased` object. -Below we join to the ``Address`` entity twice, to locate a user who has two -distinct email addresses at the same time: - -.. sourcecode:: python+sql - - >>> from sqlalchemy.orm import aliased - >>> adalias1 = aliased(Address) - >>> adalias2 = aliased(Address) - {sql}>>> for username, email1, email2 in \ - ... session.query(User.name, adalias1.email_address, adalias2.email_address).\ - ... join(User.addresses.of_type(adalias1)).\ - ... join(User.addresses.of_type(adalias2)).\ - ... filter(adalias1.email_address=='jack@google.com').\ - ... filter(adalias2.email_address=='j25@yahoo.com'): - ... print(username, email1, email2) - SELECT users.name AS users_name, - addresses_1.email_address AS addresses_1_email_address, - addresses_2.email_address AS addresses_2_email_address - FROM users JOIN addresses AS addresses_1 - ON users.id = addresses_1.user_id - JOIN addresses AS addresses_2 - ON users.id = addresses_2.user_id - WHERE addresses_1.email_address = ? - AND addresses_2.email_address = ? - [...] ('jack@google.com', 'j25@yahoo.com') - {stop}jack jack@google.com j25@yahoo.com - -In addition to using the :meth:`_orm.PropComparator.of_type` method, it is -common to see the :meth:`_orm.Query.join` method joining to a specific -target by indicating it separately:: - - # equivalent to query.join(User.addresses.of_type(adalias1)) - q = query.join(adalias1, User.addresses) - -Using Subqueries ----------------- - -The :class:`~sqlalchemy.orm.query.Query` is suitable for generating statements -which can be used as subqueries. Suppose we wanted to load ``User`` objects -along with a count of how many ``Address`` records each user has. The best way -to generate SQL like this is to get the count of addresses grouped by user -ids, and JOIN to the parent. In this case we use a LEFT OUTER JOIN so that we -get rows back for those users who don't have any addresses, e.g.:: - - SELECT users.*, adr_count.address_count FROM users LEFT OUTER JOIN - (SELECT user_id, count(*) AS address_count - FROM addresses GROUP BY user_id) AS adr_count - ON users.id=adr_count.user_id - -Using the :class:`~sqlalchemy.orm.query.Query`, we build a statement like this -from the inside out. The ``statement`` accessor returns a SQL expression -representing the statement generated by a particular -:class:`~sqlalchemy.orm.query.Query` - this is an instance of a :func:`_expression.select` -construct, which are described in :ref:`sqlexpression_toplevel`:: - - >>> from sqlalchemy.sql import func - >>> stmt = session.query(Address.user_id, func.count('*').\ - ... label('address_count')).\ - ... group_by(Address.user_id).subquery() - -The ``func`` keyword generates SQL functions, and the ``subquery()`` method on -:class:`~sqlalchemy.orm.query.Query` produces a SQL expression construct -representing a SELECT statement embedded within an alias (it's actually -shorthand for ``query.statement.alias()``). - -Once we have our statement, it behaves like a -:class:`~sqlalchemy.schema.Table` construct, such as the one we created for -``users`` at the start of this tutorial. The columns on the statement are -accessible through an attribute called ``c``: - -.. sourcecode:: python+sql - - {sql}>>> for u, count in session.query(User, stmt.c.address_count).\ - ... outerjoin(stmt, User.id==stmt.c.user_id).order_by(User.id): - ... print(u, count) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - anon_1.address_count AS anon_1_address_count - FROM users LEFT OUTER JOIN - (SELECT addresses.user_id AS user_id, count(?) AS address_count - FROM addresses GROUP BY addresses.user_id) AS anon_1 - ON users.id = anon_1.user_id - ORDER BY users.id - [...] ('*',) - {stop} None - None - None - None - 2 - -Selecting Entities from Subqueries ----------------------------------- - -Above, we just selected a result that included a column from a subquery. What -if we wanted our subquery to map to an entity ? For this we use ``aliased()`` -to associate an "alias" of a mapped class to a subquery: - -.. sourcecode:: python+sql - - {sql}>>> stmt = session.query(Address).\ - ... filter(Address.email_address != 'j25@yahoo.com').\ - ... subquery() - >>> addr_alias = aliased(Address, stmt) - >>> for user, address in session.query(User, addr_alias).\ - ... join(addr_alias, User.addresses): - ... print(user) - ... print(address) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - anon_1.id AS anon_1_id, - anon_1.email_address AS anon_1_email_address, - anon_1.user_id AS anon_1_user_id - FROM users JOIN - (SELECT addresses.id AS id, - addresses.email_address AS email_address, - addresses.user_id AS user_id - FROM addresses - WHERE addresses.email_address != ?) AS anon_1 - ON users.id = anon_1.user_id - [...] ('j25@yahoo.com',) - {stop} - - -Using EXISTS ------------- - -The EXISTS keyword in SQL is a boolean operator which returns True if the -given expression contains any rows. It may be used in many scenarios in place -of joins, and is also useful for locating rows which do not have a -corresponding row in a related table. - -There is an explicit EXISTS construct, which looks like this: - -.. sourcecode:: python+sql - - >>> from sqlalchemy.sql import exists - >>> stmt = exists().where(Address.user_id==User.id) - {sql}>>> for name, in session.query(User.name).filter(stmt): - ... print(name) - SELECT users.name AS users_name - FROM users - WHERE EXISTS (SELECT * - FROM addresses - WHERE addresses.user_id = users.id) - [...] () - {stop}jack - -The :class:`~sqlalchemy.orm.query.Query` features several operators which make -usage of EXISTS automatically. Above, the statement can be expressed along the -``User.addresses`` relationship using :meth:`~.RelationshipProperty.Comparator.any`: - -.. sourcecode:: python+sql - - {sql}>>> for name, in session.query(User.name).\ - ... filter(User.addresses.any()): - ... print(name) - SELECT users.name AS users_name - FROM users - WHERE EXISTS (SELECT 1 - FROM addresses - WHERE users.id = addresses.user_id) - [...] () - {stop}jack - -:meth:`~.RelationshipProperty.Comparator.any` takes criterion as well, to limit the rows matched: - -.. sourcecode:: python+sql - - {sql}>>> for name, in session.query(User.name).\ - ... filter(User.addresses.any(Address.email_address.like('%google%'))): - ... print(name) - SELECT users.name AS users_name - FROM users - WHERE EXISTS (SELECT 1 - FROM addresses - WHERE users.id = addresses.user_id AND addresses.email_address LIKE ?) - [...] ('%google%',) - {stop}jack - -:meth:`~.RelationshipProperty.Comparator.has` is the same operator as -:meth:`~.RelationshipProperty.Comparator.any` for many-to-one relationships -(note the ``~`` operator here too, which means "NOT"): - -.. sourcecode:: python+sql - - {sql}>>> session.query(Address).\ - ... filter(~Address.user.has(User.name=='jack')).all() - SELECT addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE NOT (EXISTS (SELECT 1 - FROM users - WHERE users.id = addresses.user_id AND users.name = ?)) - [...] ('jack',) - {stop}[] - -Common Relationship Operators ------------------------------ - -Here's all the operators which build on relationships - each one -is linked to its API documentation which includes full details on usage -and behavior: - -* :meth:`~.RelationshipProperty.Comparator.__eq__` (many-to-one "equals" comparison):: - - query.filter(Address.user == someuser) - -* :meth:`~.RelationshipProperty.Comparator.__ne__` (many-to-one "not equals" comparison):: - - query.filter(Address.user != someuser) - -* IS NULL (many-to-one comparison, also uses :meth:`~.RelationshipProperty.Comparator.__eq__`):: - - query.filter(Address.user == None) - -* :meth:`~.RelationshipProperty.Comparator.contains` (used for one-to-many collections):: - - query.filter(User.addresses.contains(someaddress)) - -* :meth:`~.RelationshipProperty.Comparator.any` (used for collections):: - - query.filter(User.addresses.any(Address.email_address == 'bar')) - - # also takes keyword arguments: - query.filter(User.addresses.any(email_address='bar')) - -* :meth:`~.RelationshipProperty.Comparator.has` (used for scalar references):: - - query.filter(Address.user.has(name='ed')) - -* :meth:`_query.Query.with_parent` (used for any relationship):: - - session.query(Address).with_parent(someuser, 'addresses') - -Eager Loading -============= - -Recall earlier that we illustrated a :term:`lazy loading` operation, when -we accessed the ``User.addresses`` collection of a ``User`` and SQL -was emitted. If you want to reduce the number of queries (dramatically, in many cases), -we can apply an :term:`eager load` to the query operation. SQLAlchemy -offers three types of eager loading, two of which are automatic, and a third -which involves custom criterion. All three are usually invoked via functions known -as query options which give additional instructions to the :class:`_query.Query` on how -we would like various attributes to be loaded, via the :meth:`_query.Query.options` method. - -Selectin Load -------------- - -In this case we'd like to indicate that ``User.addresses`` should load eagerly. -A good choice for loading a set of objects as well as their related collections -is the :func:`_orm.selectinload` option, which emits a second SELECT statement -that fully loads the collections associated with the results just loaded. -The name "selectin" originates from the fact that the SELECT statement -uses an IN clause in order to locate related rows for multiple objects -at once: - -.. sourcecode:: python+sql - - >>> from sqlalchemy.orm import selectinload - {sql}>>> jack = session.query(User).\ - ... options(selectinload(User.addresses)).\ - ... filter_by(name='jack').one() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ? - [...] ('jack',) - SELECT addresses.user_id AS addresses_user_id, - addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address - FROM addresses - WHERE addresses.user_id IN (?) - ORDER BY addresses.id - [...] (5,) - {stop}>>> jack - - - >>> jack.addresses - [, ] - - -Joined Load ------------ - -The other automatic eager loading function is more well known and is called -:func:`_orm.joinedload`. This style of loading emits a JOIN, by default -a LEFT OUTER JOIN, so that the lead object as well as the related object -or collection is loaded in one step. We illustrate loading the same -``addresses`` collection in this way - note that even though the ``User.addresses`` -collection on ``jack`` is actually populated right now, the query -will emit the extra join regardless: - -.. sourcecode:: python+sql - - >>> from sqlalchemy.orm import joinedload - - {sql}>>> jack = session.query(User).\ - ... options(joinedload(User.addresses)).\ - ... filter_by(name='jack').one() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - addresses_1.id AS addresses_1_id, - addresses_1.email_address AS addresses_1_email_address, - addresses_1.user_id AS addresses_1_user_id - FROM users - LEFT OUTER JOIN addresses AS addresses_1 ON users.id = addresses_1.user_id - WHERE users.name = ? ORDER BY addresses_1.id - [...] ('jack',) - - {stop}>>> jack - - - >>> jack.addresses - [, ] - -Note that even though the OUTER JOIN resulted in two rows, we still only got -one instance of ``User`` back. This is because :class:`_query.Query` applies a "uniquing" -strategy, based on object identity, to the returned entities. This is specifically -so that joined eager loading can be applied without affecting the query results. - -While :func:`_orm.joinedload` has been around for a long time, :func:`.selectinload` -is a newer form of eager loading. :func:`.selectinload` tends to be more appropriate -for loading related collections while :func:`_orm.joinedload` tends to be better suited -for many-to-one relationships, due to the fact that only one row is loaded -for both the lead and the related object. Another form of loading, -:func:`.subqueryload`, also exists, which can be used in place of -:func:`.selectinload` when making use of composite primary keys on certain -backends. - -.. topic:: ``joinedload()`` is not a replacement for ``join()`` - - The join created by :func:`_orm.joinedload` is anonymously aliased such that - it **does not affect the query results**. An :meth:`_query.Query.order_by` - or :meth:`_query.Query.filter` call **cannot** reference these aliased - tables - so-called "user space" joins are constructed using - :meth:`_query.Query.join`. The rationale for this is that :func:`_orm.joinedload` is only - applied in order to affect how related objects or collections are loaded - as an optimizing detail - it can be added or removed with no impact - on actual results. See the section :ref:`zen_of_eager_loading` for - a detailed description of how this is used. - -Explicit Join + Eagerload -------------------------- - -A third style of eager loading is when we are constructing a JOIN explicitly in -order to locate the primary rows, and would like to additionally apply the extra -table to a related object or collection on the primary object. This feature -is supplied via the :func:`_orm.contains_eager` function, and is most -typically useful for pre-loading the many-to-one object on a query that needs -to filter on that same object. Below we illustrate loading an ``Address`` -row as well as the related ``User`` object, filtering on the ``User`` named -"jack" and using :func:`_orm.contains_eager` to apply the "user" columns to the ``Address.user`` -attribute: - -.. sourcecode:: python+sql - - >>> from sqlalchemy.orm import contains_eager - {sql}>>> jacks_addresses = session.query(Address).\ - ... join(Address.user).\ - ... filter(User.name=='jack').\ - ... options(contains_eager(Address.user)).\ - ... all() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname, - addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses JOIN users ON users.id = addresses.user_id - WHERE users.name = ? - [...] ('jack',) - - {stop}>>> jacks_addresses - [, ] - - >>> jacks_addresses[0].user - - -For more information on eager loading, including how to configure various forms -of loading by default, see the section :doc:`/orm/loading_relationships`. - -Deleting -======== - -Let's try to delete ``jack`` and see how that goes. We'll mark the object as deleted -in the session, then we'll issue a ``count`` query to see that no rows remain: - -.. sourcecode:: python+sql - - >>> session.delete(jack) - {sql}>>> session.query(User).filter_by(name='jack').count() - UPDATE addresses SET user_id=? WHERE addresses.id = ? - [...] ((None, 1), (None, 2)) - DELETE FROM users WHERE users.id = ? - [...] (5,) - SELECT count(*) AS count_1 - FROM (SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ?) AS anon_1 - [...] ('jack',) - {stop}0 - -So far, so good. How about Jack's ``Address`` objects ? - -.. sourcecode:: python+sql - - {sql}>>> session.query(Address).filter( - ... Address.email_address.in_(['jack@google.com', 'j25@yahoo.com']) - ... ).count() - SELECT count(*) AS count_1 - FROM (SELECT addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE addresses.email_address IN (?, ?)) AS anon_1 - [...] ('jack@google.com', 'j25@yahoo.com') - {stop}2 - -Uh oh, they're still there ! Analyzing the flush SQL, we can see that the -``user_id`` column of each address was set to NULL, but the rows weren't -deleted. SQLAlchemy doesn't assume that deletes cascade, you have to tell it -to do so. - -.. _tutorial_delete_cascade: - -Configuring delete/delete-orphan Cascade ----------------------------------------- - -We will configure **cascade** options on the ``User.addresses`` relationship -to change the behavior. While SQLAlchemy allows you to add new attributes and -relationships to mappings at any point in time, in this case the existing -relationship needs to be removed, so we need to tear down the mappings -completely and start again - we'll close the :class:`.Session`:: - - >>> session.close() - ROLLBACK - - -and use a new :func:`.declarative_base`:: - - >>> Base = declarative_base() - -Next we'll declare the ``User`` class, adding in the ``addresses`` relationship -including the cascade configuration (we'll leave the constructor out too):: - - >>> class User(Base): - ... __tablename__ = 'users' - ... - ... id = Column(Integer, primary_key=True) - ... name = Column(String) - ... fullname = Column(String) - ... nickname = Column(String) - ... - ... addresses = relationship("Address", back_populates='user', - ... cascade="all, delete, delete-orphan") - ... - ... def __repr__(self): - ... return "" % ( - ... self.name, self.fullname, self.nickname) - -Then we recreate ``Address``, noting that in this case we've created -the ``Address.user`` relationship via the ``User`` class already:: - - >>> class Address(Base): - ... __tablename__ = 'addresses' - ... id = Column(Integer, primary_key=True) - ... email_address = Column(String, nullable=False) - ... user_id = Column(Integer, ForeignKey('users.id')) - ... user = relationship("User", back_populates="addresses") - ... - ... def __repr__(self): - ... return "" % self.email_address - -Now when we load the user ``jack`` (below using :meth:`_query.Query.get`, -which loads by primary key), removing an address from the -corresponding ``addresses`` collection will result in that ``Address`` -being deleted: - -.. sourcecode:: python+sql - - # load Jack by primary key - {sql}>>> jack = session.get(User, 5) - BEGIN (implicit) - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.id = ? - [...] (5,) - {stop} - - # remove one Address (lazy load fires off) - {sql}>>> del jack.addresses[1] - SELECT addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE ? = addresses.user_id - [...] (5,) - {stop} - - # only one address remains - {sql}>>> session.query(Address).filter( - ... Address.email_address.in_(['jack@google.com', 'j25@yahoo.com']) - ... ).count() - DELETE FROM addresses WHERE addresses.id = ? - [...] (2,) - SELECT count(*) AS count_1 - FROM (SELECT addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE addresses.email_address IN (?, ?)) AS anon_1 - [...] ('jack@google.com', 'j25@yahoo.com') - {stop}1 - -Deleting Jack will delete both Jack and the remaining ``Address`` associated -with the user: - -.. sourcecode:: python+sql - - >>> session.delete(jack) - - {sql}>>> session.query(User).filter_by(name='jack').count() - DELETE FROM addresses WHERE addresses.id = ? - [...] (1,) - DELETE FROM users WHERE users.id = ? - [...] (5,) - SELECT count(*) AS count_1 - FROM (SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ?) AS anon_1 - [...] ('jack',) - {stop}0 - - {sql}>>> session.query(Address).filter( - ... Address.email_address.in_(['jack@google.com', 'j25@yahoo.com']) - ... ).count() - SELECT count(*) AS count_1 - FROM (SELECT addresses.id AS addresses_id, - addresses.email_address AS addresses_email_address, - addresses.user_id AS addresses_user_id - FROM addresses - WHERE addresses.email_address IN (?, ?)) AS anon_1 - [...] ('jack@google.com', 'j25@yahoo.com') - {stop}0 - -.. topic:: More on Cascades - - Further detail on configuration of cascades is at :ref:`unitofwork_cascades`. - The cascade functionality can also integrate smoothly with - the ``ON DELETE CASCADE`` functionality of the relational database. - See :ref:`passive_deletes` for details. - -.. _orm_tutorial_many_to_many: - -Building a Many To Many Relationship -==================================== - -We're moving into the bonus round here, but lets show off a many-to-many -relationship. We'll sneak in some other features too, just to take a tour. -We'll make our application a blog application, where users can write -``BlogPost`` items, which have ``Keyword`` items associated with them. - -For a plain many-to-many, we need to create an un-mapped :class:`_schema.Table` construct -to serve as the association table. This looks like the following:: - - >>> from sqlalchemy import Table, Text - >>> # association table - >>> post_keywords = Table('post_keywords', Base.metadata, - ... Column('post_id', ForeignKey('posts.id'), primary_key=True), - ... Column('keyword_id', ForeignKey('keywords.id'), primary_key=True) - ... ) - -Above, we can see declaring a :class:`_schema.Table` directly is a little different -than declaring a mapped class. :class:`_schema.Table` is a constructor function, so -each individual :class:`_schema.Column` argument is separated by a comma. The -:class:`_schema.Column` object is also given its name explicitly, rather than it being -taken from an assigned attribute name. - -Next we define ``BlogPost`` and ``Keyword``, using complementary -:func:`_orm.relationship` constructs, each referring to the ``post_keywords`` -table as an association table:: - - >>> class BlogPost(Base): - ... __tablename__ = 'posts' - ... - ... id = Column(Integer, primary_key=True) - ... user_id = Column(Integer, ForeignKey('users.id')) - ... headline = Column(String(255), nullable=False) - ... body = Column(Text) - ... - ... # many to many BlogPost<->Keyword - ... keywords = relationship('Keyword', - ... secondary=post_keywords, - ... back_populates='posts') - ... - ... def __init__(self, headline, body, author): - ... self.author = author - ... self.headline = headline - ... self.body = body - ... - ... def __repr__(self): - ... return "BlogPost(%r, %r, %r)" % (self.headline, self.body, self.author) - - - >>> class Keyword(Base): - ... __tablename__ = 'keywords' - ... - ... id = Column(Integer, primary_key=True) - ... keyword = Column(String(50), nullable=False, unique=True) - ... posts = relationship('BlogPost', - ... secondary=post_keywords, - ... back_populates='keywords') - ... - ... def __init__(self, keyword): - ... self.keyword = keyword - -.. note:: - - The above class declarations illustrate explicit ``__init__()`` methods. - Remember, when using Declarative, it's optional! - -Above, the many-to-many relationship is ``BlogPost.keywords``. The defining -feature of a many-to-many relationship is the ``secondary`` keyword argument -which references a :class:`~sqlalchemy.schema.Table` object representing the -association table. This table only contains columns which reference the two -sides of the relationship; if it has *any* other columns, such as its own -primary key, or foreign keys to other tables, SQLAlchemy requires a different -usage pattern called the "association object", described at -:ref:`association_pattern`. - -We would also like our ``BlogPost`` class to have an ``author`` field. We will -add this as another bidirectional relationship, except one issue we'll have is -that a single user might have lots of blog posts. When we access -``User.posts``, we'd like to be able to filter results further so as not to -load the entire collection. For this we use a setting accepted by -:func:`~sqlalchemy.orm.relationship` called ``lazy='dynamic'``, which -configures an alternate **loader strategy** on the attribute: - -.. sourcecode:: python+sql - - >>> BlogPost.author = relationship(User, back_populates="posts") - >>> User.posts = relationship(BlogPost, back_populates="author", lazy="dynamic") - -Create new tables: - -.. sourcecode:: python+sql - - {sql}>>> Base.metadata.create_all(engine) - BEGIN... - CREATE TABLE keywords ( - id INTEGER NOT NULL, - keyword VARCHAR(50) NOT NULL, - PRIMARY KEY (id), - UNIQUE (keyword) - ) - [...] () - CREATE TABLE posts ( - id INTEGER NOT NULL, - user_id INTEGER, - headline VARCHAR(255) NOT NULL, - body TEXT, - PRIMARY KEY (id), - FOREIGN KEY(user_id) REFERENCES users (id) - ) - [...] () - CREATE TABLE post_keywords ( - post_id INTEGER NOT NULL, - keyword_id INTEGER NOT NULL, - PRIMARY KEY (post_id, keyword_id), - FOREIGN KEY(post_id) REFERENCES posts (id), - FOREIGN KEY(keyword_id) REFERENCES keywords (id) - ) - [...] () - COMMIT - -Usage is not too different from what we've been doing. Let's give Wendy some blog posts: - -.. sourcecode:: python+sql - - {sql}>>> wendy = session.query(User).\ - ... filter_by(name='wendy').\ - ... one() - SELECT users.id AS users_id, - users.name AS users_name, - users.fullname AS users_fullname, - users.nickname AS users_nickname - FROM users - WHERE users.name = ? - [...] ('wendy',) - {stop} - >>> post = BlogPost("Wendy's Blog Post", "This is a test", wendy) - >>> session.add(post) - -We're storing keywords uniquely in the database, but we know that we don't -have any yet, so we can just create them: - -.. sourcecode:: python+sql - - >>> post.keywords.append(Keyword('wendy')) - >>> post.keywords.append(Keyword('firstpost')) - -We can now look up all blog posts with the keyword 'firstpost'. We'll use the -``any`` operator to locate "blog posts where any of its keywords has the -keyword string 'firstpost'": - -.. sourcecode:: python+sql - - {sql}>>> session.query(BlogPost).\ - ... filter(BlogPost.keywords.any(keyword='firstpost')).\ - ... all() - INSERT INTO keywords (keyword) VALUES (?) - [...] ('wendy',) - INSERT INTO keywords (keyword) VALUES (?) - [...] ('firstpost',) - INSERT INTO posts (user_id, headline, body) VALUES (?, ?, ?) - [...] (2, "Wendy's Blog Post", 'This is a test') - INSERT INTO post_keywords (post_id, keyword_id) VALUES (?, ?) - [...] (...) - SELECT posts.id AS posts_id, - posts.user_id AS posts_user_id, - posts.headline AS posts_headline, - posts.body AS posts_body - FROM posts - WHERE EXISTS (SELECT 1 - FROM post_keywords, keywords - WHERE posts.id = post_keywords.post_id - AND keywords.id = post_keywords.keyword_id - AND keywords.keyword = ?) - [...] ('firstpost',) - {stop}[BlogPost("Wendy's Blog Post", 'This is a test', )] - -If we want to look up posts owned by the user ``wendy``, we can tell -the query to narrow down to that ``User`` object as a parent: - -.. sourcecode:: python+sql - - {sql}>>> session.query(BlogPost).\ - ... filter(BlogPost.author==wendy).\ - ... filter(BlogPost.keywords.any(keyword='firstpost')).\ - ... all() - SELECT posts.id AS posts_id, - posts.user_id AS posts_user_id, - posts.headline AS posts_headline, - posts.body AS posts_body - FROM posts - WHERE ? = posts.user_id AND (EXISTS (SELECT 1 - FROM post_keywords, keywords - WHERE posts.id = post_keywords.post_id - AND keywords.id = post_keywords.keyword_id - AND keywords.keyword = ?)) - [...] (2, 'firstpost') - {stop}[BlogPost("Wendy's Blog Post", 'This is a test', )] - -Or we can use Wendy's own ``posts`` relationship, which is a "dynamic" -relationship, to query straight from there: - -.. sourcecode:: python+sql - - {sql}>>> wendy.posts.\ - ... filter(BlogPost.keywords.any(keyword='firstpost')).\ - ... all() - SELECT posts.id AS posts_id, - posts.user_id AS posts_user_id, - posts.headline AS posts_headline, - posts.body AS posts_body - FROM posts - WHERE ? = posts.user_id AND (EXISTS (SELECT 1 - FROM post_keywords, keywords - WHERE posts.id = post_keywords.post_id - AND keywords.id = post_keywords.keyword_id - AND keywords.keyword = ?)) - [...] (2, 'firstpost') - {stop}[BlogPost("Wendy's Blog Post", 'This is a test', )] - -Further Reference -================== - -Query Reference: :ref:`query_api_toplevel` - -Mapper Reference: :ref:`mapper_config_toplevel` - -Relationship Reference: :ref:`relationship_config_toplevel` - -Session Reference: :doc:`/orm/session` - - -.. Setup code, not for display - - >>> session.close() - ROLLBACK diff --git a/doc/build/tutorial/index.rst b/doc/build/tutorial/index.rst index cb6c2feae3..ecf45f5aa3 100644 --- a/doc/build/tutorial/index.rst +++ b/doc/build/tutorial/index.rst @@ -1,4 +1,4 @@ -.. |tutorial_title| replace:: SQLAlchemy 1.4 / 2.0 Tutorial +.. |tutorial_title| replace:: SQLAlchemy 2.0 Tutorial .. |next| replace:: :doc:`engine` .. footer_topic:: |tutorial_title| @@ -9,20 +9,21 @@ .. rst-class:: orm_core -============================= -SQLAlchemy 1.4 / 2.0 Tutorial -============================= +======================== +SQLAlchemy 2.0 Tutorial +======================== .. admonition:: About this document - The new SQLAlchemy Tutorial is now integrated between Core and ORM and - serves as a unified introduction to SQLAlchemy as a whole. In the new - :term:`2.0 style` of working, fully available in the :ref:`1.4 release - `, the ORM now uses Core-style querying with the + The SQLAlchemy Tutorial for version 2.0, first published as a preview + within the 1.4 documentation, is integrated between the Core and ORM + components of SQLAlchemy and serves as a unified introduction to SQLAlchemy + as a whole. For users of SQLAlchemy within the 1.x series, in the + :term:`2.0 style` of working, the ORM uses Core-style querying with the :func:`_sql.select` construct, and transactional semantics between Core - connections and ORM sessions are equivalent. Take note of the blue - border styles for each section, that will tell you how "ORM-ish" a - particular topic is! + connections and ORM sessions are equivalent. Take note of the blue border + styles for each section, that will tell you how "ORM-ish" a particular + topic is! Users who are already familiar with SQLAlchemy, and especially those looking to migrate existing applications to work under SQLAlchemy 2.0 @@ -132,33 +133,16 @@ the reader is invited to work with the code examples given in real time with their own Python interpreter. If running the examples, it is advised that the reader performs a quick check to -verify that we are on **version 1.4** of SQLAlchemy: +verify that we are on **version 2.0** of SQLAlchemy: .. sourcecode:: pycon+sql >>> import sqlalchemy >>> sqlalchemy.__version__ # doctest: +SKIP - 1.4.0 + 2.0.0 .. rst-class:: core-header, orm-dependency -A Note on the Future ---------------------- - -This tutorial describes a new API that's released in SQLAlchemy 1.4 known -as :term:`2.0 style`. The purpose of the 2.0-style API is to provide forwards -compatibility with :ref:`SQLAlchemy 2.0 `, which is -planned as the next generation of SQLAlchemy. - -In order to provide the full 2.0 API, a new flag called ``future`` will be -used, which will be seen as the tutorial describes the :class:`_engine.Engine` -and :class:`_orm.Session` objects. These flags fully enable 2.0-compatibility -mode and allow the code in the tutorial to proceed fully. When using the -``future`` flag with the :func:`_sa.create_engine` function, the object -returned is a subclass of :class:`sqlalchemy.engine.Engine` described as -:class:`sqlalchemy.future.Engine`. This tutorial will be referring to -:class:`sqlalchemy.future.Engine`. - diff --git a/lib/sqlalchemy/__init__.py b/lib/sqlalchemy/__init__.py index ad6d96fdd3..dc49690a50 100644 --- a/lib/sqlalchemy/__init__.py +++ b/lib/sqlalchemy/__init__.py @@ -130,7 +130,7 @@ from .types import VARBINARY from .types import VARCHAR -__version__ = "1.4.27" +__version__ = "2.0.0b1" def __go(lcls): diff --git a/setup.cfg b/setup.cfg index f432561b18..d36585a9ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,10 +18,7 @@ classifiers = License :: OSI Approved :: MIT License Operating System :: OS Independent Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -35,7 +32,7 @@ project_urls = [options] packages = find: -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* +python_requires = >=3.7 package_dir = =lib diff --git a/test/base/test_tutorials.py b/test/base/test_tutorials.py index 494b8a0f67..f2b577216b 100644 --- a/test/base/test_tutorials.py +++ b/test/base/test_tutorials.py @@ -6,7 +6,6 @@ import os import re import sys -from sqlalchemy import testing from sqlalchemy.testing import config from sqlalchemy.testing import fixtures @@ -102,13 +101,6 @@ class DocTest(fixtures.TestBase): "tutorial/orm_related_objects.rst", ) - def test_orm(self): - self._run_doctest("orm/tutorial.rst") - - @testing.emits_warning() - def test_core(self): - self._run_doctest("core/tutorial.rst") - def test_core_operators(self): self._run_doctest("core/operators.rst") diff --git a/test/orm/test_cascade.py b/test/orm/test_cascade.py index cd7e7c111a..3617e3a5d5 100644 --- a/test/orm/test_cascade.py +++ b/test/orm/test_cascade.py @@ -1408,12 +1408,6 @@ class NoSaveCascadeFlushTest(_fixtures.FixtureTest): with testing.expect_deprecated( '"Address" object is being merged into a Session along ' 'the backref cascade path for relationship "User.addresses"' - # link added to this specific warning - r".*Background on this error at: " - r"https://sqlalche.me/e/14/s9r1" - # link added to all RemovedIn20Warnings - r".*Background on SQLAlchemy 2.0 at: " - r"https://sqlalche.me/e/b8d9" ): a1.user = u1 sess.add(a1)