From bc1147cae5c246aa2858c275988262b1a60cdabe Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 8 Nov 2011 14:05:37 -0800 Subject: [PATCH] docs --- alembic/command.py | 5 + alembic/context.py | 7 +- alembic/op.py | 4 +- alembic/util.py | 77 +++++++++ docs/build/api.rst | 63 +++++++- docs/build/conf.py | 7 +- docs/build/index.rst | 3 +- docs/build/ops.rst | 2 + docs/build/tutorial.rst | 342 ++++++++++++++++++++++++++++++++++++++-- 9 files changed, 491 insertions(+), 19 deletions(-) diff --git a/alembic/command.py b/alembic/command.py index c0743ae2..0c489080 100644 --- a/alembic/command.py +++ b/alembic/command.py @@ -102,6 +102,11 @@ def branches(config): for sc in script.walk_revisions(): if sc.is_branch_point: print sc + for rev in sc.nextrev: + print "%s -> %s" % ( + " " * len(str(sc.down_revision)), + script._get_rev(rev) + ) def current(config): """Display the current revision for each database.""" diff --git a/alembic/context.py b/alembic/context.py index bdb67fb8..5ba5ea1c 100644 --- a/alembic/context.py +++ b/alembic/context.py @@ -2,12 +2,11 @@ from alembic import util from sqlalchemy import MetaData, Table, Column, String, literal_column, \ text from sqlalchemy import schema, create_engine -from sqlalchemy.util import importlater from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.expression import _BindParamClause import logging -base = importlater("alembic.ddl", "base") +base = util.importlater("alembic.ddl", "base") log = logging.getLogger(__name__) class ContextMeta(type): @@ -215,6 +214,10 @@ def configure_connection(connection): DefaultContext)(connection, **_context_opts) def run_migrations(**kw): + """Run migrations as determined by the current command line configuration + as well as versioning information present (or not) in the current + database connection (if one is present). + """ _context.run_migrations(**kw) def get_context(): diff --git a/alembic/op.py b/alembic/op.py index beea769d..2e5e74f5 100644 --- a/alembic/op.py +++ b/alembic/op.py @@ -1,7 +1,9 @@ from alembic import util from alembic.context import get_context from sqlalchemy.types import NULLTYPE -from sqlalchemy import schema +from sqlalchemy import schema, sql + +util.importlater.resolve_all() __all__ = [ 'alter_column', diff --git a/alembic/util.py b/alembic/util.py index cda77c2d..51615b3a 100644 --- a/alembic/util.py +++ b/alembic/util.py @@ -98,3 +98,80 @@ class memoized_property(object): return None obj.__dict__[self.__name__] = result = self.fget(obj) return result + +class importlater(object): + """Deferred import object. + + e.g.:: + + somesubmod = importlater("mypackage.somemodule", "somesubmod") + + is equivalent to:: + + from mypackage.somemodule import somesubmod + + except evaluted upon attribute access to "somesubmod". + + importlater() currently requires that resolve_all() be + called, typically at the bottom of a package's __init__.py. + This is so that __import__ still called only at + module import time, and not potentially within + a non-main thread later on. + + """ + + _unresolved = set() + + def __init__(self, path, addtl=None): + self._il_path = path + self._il_addtl = addtl + importlater._unresolved.add(self) + + @classmethod + def resolve_all(cls): + for m in list(importlater._unresolved): + m._resolve() + + @property + def _full_path(self): + if self._il_addtl: + return self._il_path + "." + self._il_addtl + else: + return self._il_path + + @memoized_property + def module(self): + if self in importlater._unresolved: + raise ImportError( + "importlater.resolve_all() hasn't been called") + + m = self._initial_import + if self._il_addtl: + m = getattr(m, self._il_addtl) + else: + for token in self._il_path.split(".")[1:]: + m = getattr(m, token) + return m + + def _resolve(self): + importlater._unresolved.discard(self) + if self._il_addtl: + self._initial_import = __import__( + self._il_path, globals(), locals(), + [self._il_addtl]) + else: + self._initial_import = __import__(self._il_path) + + def __getattr__(self, key): + if key == 'module': + raise ImportError("Could not resolve module %s" + % self._full_path) + try: + attr = getattr(self.module, key) + except AttributeError: + raise AttributeError( + "Module %s has no attribute '%s'" % + (self._full_path, key) + ) + self.__dict__[key] = attr + return attr diff --git a/docs/build/api.rst b/docs/build/api.rst index 42fdaab7..238137c1 100644 --- a/docs/build/api.rst +++ b/docs/build/api.rst @@ -2,5 +2,66 @@ API Details =========== -This section describes some key functions used within the migration process. +This section describes some key functions used within the migration process, particularly those referenced within +a migration environment's ``env.py`` file. +env.py Directives +================= + +.. autofunction:: sqlalchemy.engine.engine_from_config +.. autofunction:: alembic.context.configure_connection +.. autofunction:: alembic.context.run_migrations + +Internals +========= + +.. automodule:: alembic.config + :members: + :undoc-members: + +.. automodule:: alembic.command + :members: + :undoc-members: + +.. automodule:: alembic.script + :members: + :undoc-members: + +DDL Internals +------------- + +.. automodule:: alembic.ddl + :members: + :undoc-members: + +.. automodule:: alembic.ddl.base + :members: + :undoc-members: + +MySQL +^^^^^ + +.. automodule:: alembic.ddl.mysql + :members: + :undoc-members: + +MS-SQL +^^^^^^ + +.. automodule:: alembic.ddl.mssql + :members: + :undoc-members: + +Postgresql +^^^^^^^^^^ + +.. automodule:: alembic.ddl.postgresql + :members: + :undoc-members: + +SQLite +^^^^^^ + +.. automodule:: alembic.ddl.sqlite + :members: + :undoc-members: diff --git a/docs/build/conf.py b/docs/build/conf.py index f5374d8d..9f7a5c62 100644 --- a/docs/build/conf.py +++ b/docs/build/conf.py @@ -45,7 +45,7 @@ master_doc = 'index' # General information about the project. project = u'Alembic' -copyright = u'2010, Mike Bayer' +copyright = u'2010-2011, Mike Bayer' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -202,5 +202,6 @@ latex_documents = [ #latex_use_modindex = True -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +#{'python': ('http://docs.python.org/3.2', None)} + +intersphinx_mapping = {'sqla':('http://www.sqlalchemy.org/docs/', None)} diff --git a/docs/build/index.rst b/docs/build/index.rst index 805724a1..4ff3820f 100644 --- a/docs/build/index.rst +++ b/docs/build/index.rst @@ -2,7 +2,8 @@ Welcome to Alembic's documentation! =================================== -Contents: +`Alembic `_ is a lightweight database migration tool for usage +with the `SQLAlchemy `_ Database Toolkit for Python. .. toctree:: :maxdepth: 2 diff --git a/docs/build/ops.rst b/docs/build/ops.rst index 809e0727..27781a94 100644 --- a/docs/build/ops.rst +++ b/docs/build/ops.rst @@ -1,3 +1,5 @@ +.. _ops: + =================== Operation Reference =================== diff --git a/docs/build/tutorial.rst b/docs/build/tutorial.rst index 77b42dc3..79b9a8a1 100644 --- a/docs/build/tutorial.rst +++ b/docs/build/tutorial.rst @@ -2,7 +2,7 @@ Tutorial ======== -Alembic provides for the creation, management, and invocation of *change management* +`Alembic `_ provides for the creation, management, and invocation of *change management* scripts for a relational database, using `SQLAlchemy `_ as the underlying engine. This tutorial will provide a full introduction to the theory and usage of this tool. @@ -177,8 +177,8 @@ This file contains the following features: ``[formatter_*]`` - these sections are all part of Python's standard logging configuration, the mechanics of which are documented at `Configuration File Format `_. As is the case with the database connection, these directives are used directly as the - result of the ``logging.config.fileConfig()`` call present in the fully customizable - ``env.py`` script. + result of the ``logging.config.fileConfig()`` call present in the + ``env.py`` script, which you're free to modify. For starting up with just a single database and the generic configuration, setting up the SQLAlchemy URL is all that's needed:: @@ -190,12 +190,12 @@ Create a Migration Script With the environment in place we can create a new revision, using ``alembic revision``:: - $ alembic revision -m "add a column" + $ alembic revision -m "create account table" Generating /path/to/yourproject/alembic/versions/1975ea83b712.py...done -A new file 1975ea83b712.py is generated. Looking inside the file:: +A new file ``1975ea83b712.py`` is generated. Looking inside the file:: - """add a column + """create account table Revision ID: 1975ea83b712 Revises: None @@ -251,18 +251,338 @@ We can then add some directives to our script, suppose adding a new table ``acco def downgrade(): drop_table('account') -foo:: +:func:`.create_table` and :func:`.drop_table` are Alembic directives. Alembic provides +all the basic database migration operations via these directives, which are designed to be as simple and +minimalistic as possible; +there's no reliance upon existing table metadata for most of these directives. They draw upon +a global "context" that indicates how to get at a database connection (if any; migrations can +dump SQL/DDL directives to files as well) in order to invoke the command. This global +context is set up, like everything else, in the ``env.py`` script. + +An overview of all Alembic directives is at :ref:`ops`. + +Running our First Migration +=========================== + +We now want to run our migration. Assuming our database is totally clean, it's as +yet unversioned. The ``alembic upgrade`` command will run upgrade operations, proceeding +from the current database revision, in this example ``None``, to the given target revision. +We can specify ``1975ea83b712`` as the revision we'd like to upgrade to, but it's easier +in most cases just to tell it "the most recent", in this case ``head``:: + + $ alembic upgrade head + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + INFO [alembic.context] Running upgrade None -> 1975ea83b712 + +Wow that rocked ! Note that the information we see on the screen is the result of the +logging configuration set up in ``alembic.ini`` - logging the ``alembic`` stream to the +console (standard error, specifically). + +The process which occurred here included that Alembic first checked if the database had +a table called ``alembic_version``, and if not, created it. It looks in this table +for the current version, if any, and then calculates the path from this version to +the version requested, in this case ``head``, which is known to be ``1975ea83b712``. +It then invokes the ``upgrade()`` method in each file to get to the target revision. + +Running our Second Migration +============================= + +OK let's do another one so we have some things to play with. We again create a revision +file:: + + $ alembic revision -m "Add a column" + Generating /path/to/yourapp/alembic/versions/ae1027a6acf.py...done + +Let's edit this file and add a new column to the ``account`` table:: + + """Add a column + + Revision ID: ae1027a6acf + Revises: 1975ea83b712 + Create Date: 2011-11-08 12:37:36.714947 + + """ + + # downgrade revision identifier, used by Alembic. + down_revision = '1975ea83b712' + + from alembic.op import * + from sqlalchemy import DateTime, Column def upgrade(): - add_column('accounts', - Column('account_id', INTEGER, ForeignKey('accounts.id')) + add_column('account', Column('last_transaction_date', DateTime)) + + def downgrade(): + drop_column('account', 'last_transaction_date') + +Running again to ``head``:: + + $ alembic upgrade head + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + INFO [alembic.context] Running upgrade 1975ea83b712 -> ae1027a6acf + +We've now added the ``last_transaction_date`` column to the database. + +Getting Information +=================== + +With a few revisions present we can get some information about the state of things. + +First we can view the current revision:: + + $ alembic current + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + Current revision for postgresql://scott:XXXXX@localhost/test: 1975ea83b712 -> ae1027a6acf (head), Add a column + +We can also view history:: + + $ alembic history + + 1975ea83b712 -> ae1027a6acf (head), Add a column + None -> 1975ea83b712, empty message + +Downgrading +=========== + +We can illustrate a downgrade back to nothing, by calling ``alembic downgrade`` back +to the beginning, which in Alembic is called ``base``:: + + $ alembic downgrade base + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + INFO [alembic.context] Running downgrade ae1027a6acf -> 1975ea83b712 + INFO [alembic.context] Running downgrade 1975ea83b712 -> None + +Back to nothing - and up again:: + + $ alembic upgrade head + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + INFO [alembic.context] Running upgrade None -> 1975ea83b712 + INFO [alembic.context] Running upgrade 1975ea83b712 -> ae1027a6acf + +Auto Generating Migrations +=========================== + +.. note:: this functionality is not yet implemented. Specific details here + are subject to change. + +Alembic can view the status of the database and compare against the table metadata +in the application, generating the "obvious" migrations based on a comparison. This +is achieved using the ``--autogenerate`` option to the ``alembic`` command. + +To use autogenerate, we first need to modify our ``env.py`` so that it gets access +to a table metadata object that contains the target. Suppose our application +has a `declarative base `_ +in ``myapp.mymodel``. This base contains a :class:`~sqlalchemy.schema.MetaData` object which +contains :class:`~sqlalchemy.schema.Table` objects defining our database. We make sure this +is loaded in ``env.py`` and then passed to :func:`.context.configure_connection` via +``use_metadata``:: + + from myapp.mymodel import Base + + connection = engine.connect() + context.configure_connection(connection, use_metadata=Base.metadata) + trans = connection.begin() + try: + context.run_migrations() + trans.commit() + except: + trans.rollback() + raise + +We then create an upgrade file in the usual way adding ``--autogenerate``. Suppose +our :class:`~sqlalchemy.schema.MetaData` contained a definition for the ``account`` table, +and the database did not. We'd get output like:: + + $ alembic revision --autogenerate -m "Added account table" + INFO [alembic.context] Detected added table 'account' + Generating /Users/classic/Desktop/tmp/alembic/versions/27c6a30d7c24.py...done + +We can then view our file ``27c6a30d7c24.py`` and see that a rudimentary migration +is already present:: + + """empty message + + Revision ID: 27c6a30d7c24 + Revises: None + Create Date: 2011-11-08 11:40:27.089406 + + """ + + # downgrade revision identifier, used by Alembic. + down_revision = None + + from alembic.op import * + + import sqlalchemy as sa + + def upgrade(): + create_table( + 'account', + sa.Column('id', sa.INTEGER, primary_key=True), + sa.Column('name', sa.VARCHAR(50), nullable=False), + sa.Column('description', sa.VARCHAR(200)), + sa.Column('last_transaction_date', sa.DATETIME) ) def downgrade(): - drop_column('organization', 'account_id') - drop_table("accounts") + drop_table("account") + +The migration hasn't actually run yet, of course. We do that via the usual ``upgrade`` +command. We should also go into our migration file and alter it as needed, including +adjustments to the directives as well as the addition of other directives which these may +be dependent on - specifically data changes in between creates/alters/drops. The autogenerate +feature can currently detect: + +* Table additions, removals. +* Column additions, removals +* Change of column type, nullable status + +Autogenerate can *not* detect: + +* Changes of table name. These will come out as an add/drop of two different + tables, and should be hand-edited into a name change instead. +* Changes of column name. Like table name changes, these are detected as + a column add/drop pair, which is not at all the same as a name change. +* Constraint addition/removal. This is potentially possible but is not + yet implemented. + + +Generating SQL Scripts +====================== + +A major capability of Alembic is to generate migrations as SQL scripts, instead of running +them against the database. This is a critical feature when working in large organizations +where access to DDL is restricted, and SQL scripts must be handed off to DBAs. Alembic makes +this easy via the ``--sql`` option passed to any ``upgrade`` or ``downgrade`` command. We +can, for example, generate a script that revises up to rev ``ae1027a6acf``:: + + $ alembic upgrade ae1027a6acf --sql + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + BEGIN; + + INFO [alembic.context] Running upgrade None -> 1975ea83b712 + + CREATE TABLE account ( + id SERIAL NOT NULL, + name VARCHAR(50) NOT NULL, + description VARCHAR(200), + PRIMARY KEY (id) + ) + + ; + INFO [alembic.context] Running upgrade 1975ea83b712 -> ae1027a6acf + ALTER TABLE account ADD COLUMN last_transaction_date TIMESTAMP WITHOUT TIME ZONE; + INSERT INTO alembic_version (version_num) VALUES ('ae1027a6acf'); + COMMIT; + +While the logging configuration dumped to standard error, the actual script was dumped to standard output - +so typically we'd be using output redirection to generate a script:: + + $ alembic upgrade ae1027a6acf --sql > migration.sql + +Generating on a Range +--------------------- + +Notice that our migration script started at the base - this is the default when using the ``--sql`` +operation, which does not otherwise make usage of a database connection, so does not retrieve +any starting version. We usually will want +to specify a start/end version. This is allowed when using the ``--sql`` option only +using the ``start:end`` syntax:: + + $ alembic upgrade 1975ea83b712:ae1027a6acf --sql > migration.sql + +Writing Migration Scripts to Support Script Generation +------------------------------------------------------ + +The challenge of SQL script generation is that the scripts we generate can't rely upon +any client/server database access. This means a migration script that pulls some rows +into memory via a ``SELECT`` statement will not work in ``--sql`` mode. It's also +important that the Alembic directives, all of which are designed specifically to work +in both "live execution" as well as "offline SQL generation" mode, are used. + + +Working with Branches +===================== + +A *branch* describes when a source tree is broken up into two versions representing +two independent sets of changes. The challenge of a branch is to *merge* the +branches into a single series of changes. Alembic's GUID-based version number scheme +allows branches to be reconciled. + +Consider if we merged into our source repository another branch which contained +a revision for another table called ``shopping_cart``. This revision was made +against our first Alembic revision, the one that generated ``account``. After +loading the second source tree in, a new file ``27c6a30d7c24.py`` exists within +our ``versions`` directory. Both it, as well as ``ae1027a6acf.py``, reference +``1975ea83b712`` as the "downgrade" revision. To illustrate:: + + # main source tree: + 1975ea83b712 (add account table) -> ae1027a6acf (add a column) + + # branched source tree + 1975ea83b712 (add account table) -> 27c6a30d7c24 (add shopping cart table) + +So above we can see 1975ea83b712 is our *branch point*. The Alembic command ``branches`` +illustrates this fact:: + + $ alembic branches + None -> 1975ea83b712 (branchpoint), add account table + -> 1975ea83b712 -> 27c6a30d7c24 (head), add shopping cart table + -> 1975ea83b712 -> ae1027a6acf (head), add a column + +History shows it too, illustrating two ``head`` entries as well +as a ``branchpoint``:: + + $ alembic history + + 1975ea83b712 -> 27c6a30d7c24 (head), add shopping cart table + + 1975ea83b712 -> ae1027a6acf (head), add a column + None -> 1975ea83b712 (branchpoint), add account table + +Alembic will also refuse to run any migrations until this is resolved [TODO: alembic dumps the +whole stack, needs to return just a message]:: + + $ alembic upgrade head + INFO [alembic.context] Context class PostgresqlContext. + INFO [alembic.context] Will assume transactional DDL. + Exception: Only a single head supported so far... + +We resolve this branch by editing the files to be in a straight line. In this case we edit +``27c6a30d7c24.py`` to point to ``ae1027a6acf.py``:: + + """add shopping cart table + + Revision ID: 27c6a30d7c24 + Revises: ae1027a6acf # changed from 1975ea83b712 + Create Date: 2011-11-08 13:02:14.212810 + + """ + + # downgrade revision identifier, used by Alembic. + # changed from 1975ea83b712 + down_revision = 'ae1027a6acf' + +The ``branches`` command then shows no branches:: + + $ alembic branches + $ + +And the history is similarly linear:: + + $ alembic history + ae1027a6acf -> 27c6a30d7c24 (head), add shopping cart table + 1975ea83b712 -> ae1027a6acf, add a column + None -> 1975ea83b712, add account table +.. note:: A future command called ``splice`` will automate this process. .. _building_uptodate: -- 2.47.2