From: Mike Bayer Date: Mon, 14 Nov 2011 22:39:17 +0000 (-0500) Subject: - make start/end arguments available to environments X-Git-Tag: rel_0_1_0~60 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f576169d362694139193dc29e7d2ba9f57323809;p=thirdparty%2Fsqlalchemy%2Falembic.git - make start/end arguments available to environments - more environment functions - clean up start:end system - docs --- diff --git a/alembic/command.py b/alembic/command.py index 0ea8bdbd..ba0810e4 100644 --- a/alembic/command.py +++ b/alembic/command.py @@ -65,25 +65,46 @@ def revision(config, message=None): script = ScriptDirectory.from_config(config) script.generate_rev(util.rev_id(), message) -def upgrade(config, revision, sql=False): +def upgrade(config, revision, sql=False, tag=None): """Upgrade to a later version.""" script = ScriptDirectory.from_config(config) - context.opts( + + starting_rev = None + if ":" in revision: + if not sql: + raise util.CommandError("Range revision not allowed") + starting_rev, revision = revision.split(':', 2) + context._opts( config, - fn = functools.partial(script.upgrade_from, sql, revision), - as_sql = sql + script, + fn = functools.partial(script.upgrade_from, revision), + as_sql = sql, + starting_rev = starting_rev, + destination_rev = revision, + tag = tag ) script.run_env() -def downgrade(config, revision, sql=False): +def downgrade(config, revision, sql=False, tag=None): """Revert to a previous version.""" script = ScriptDirectory.from_config(config) - context.opts( + + starting_rev = None + if ":" in revision: + if not sql: + raise util.CommandError("Range revision not allowed") + starting_rev, revision = revision.split(':', 2) + + context._opts( config, - fn = functools.partial(script.downgrade_to, sql, revision), + script, + fn = functools.partial(script.downgrade_to, revision), as_sql = sql, + starting_rev = starting_rev, + destination_rev = revision, + tag = tag ) script.run_env() @@ -119,13 +140,14 @@ def current(config): script._get_rev(rev)) return [] - context.opts( + context._opts( config, + script, fn = display_version ) script.run_env() -def stamp(config, revision, sql=False): +def stamp(config, revision, sql=False, tag=None): """'stamp' the revision table with the given revision; don't run any migrations.""" @@ -140,10 +162,13 @@ def stamp(config, revision, sql=False): dest = dest.revision context.get_context()._update_current_rev(current, dest) return [] - context.opts( + context._opts( config, + script, fn = do_stamp, as_sql = sql, + destination_rev = revision, + tag = tag ) script.run_env() diff --git a/alembic/config.py b/alembic/config.py index dc1b694f..26750a98 100644 --- a/alembic/config.py +++ b/alembic/config.py @@ -22,24 +22,62 @@ class Config(object): self.config_file_name = file_ self.config_ini_section = ini_section + config_file_name = None + """Filesystem path to the .ini file in use.""" + + config_ini_section = None + """Name of the config file section to read basic configuration + from. Defaults to ``alembic``, that is the ``[alembic]`` section + of the .ini file. This value is modified using the ``-n/--name`` + option to the Alembic runnier. + + """ + @util.memoized_property def file_config(self): - file_config = ConfigParser.ConfigParser() + """Return the underlying :class:`ConfigParser` object. + + Direct access to the .ini file is available here, + though the :meth:`.Config.get_section` and + :meth:`.Config.get_main_option` + methods provide a possibly simpler interface. + """ + + file_config = ConfigParser.ConfigParser({ + 'here': + os.path.abspath(os.path.dirname(self.config_file_name))}) file_config.read([self.config_file_name]) return file_config def get_template_directory(self): + """Return the directory where Alembic setup templates are found. + + This method is used by the alembic ``init`` and ``list_templates`` + commands. + + """ return os.path.join(package_dir, 'templates') def get_section(self, name): + """Return all the configuration options from a given .ini file section + as a dictionary. + + """ return dict(self.file_config.items(name)) def get_main_option(self, name, default=None): + """Return an option from the 'main' section of the .ini file. + + This defaults to being a key from the ``[alembic]`` + section, unless the ``-n/--name`` flag were used to + indicate a different section. + + """ if not self.file_config.has_section(self.config_ini_section): util.err("No config file %r found, or file has no " "'[%s]' section" % (self.config_file_name, self.config_ini_section)) - if self.file_config.get(self.config_ini_section, name): + if self.file_config.has_option(self.config_ini_section, name): return self.file_config.get(self.config_ini_section, name) else: return default @@ -61,7 +99,13 @@ def main(argv): parser.add_argument("--sql", action="store_true", help="Don't emit SQL to database - dump to " - "standard output instead") + "standard output/file instead") + if 'tag' in kwargs: + parser.add_argument("--tag", + type=str, + help="Arbitrary 'tag' name - can be used by " + "custom env.py scripts.") + # TODO: # --dialect - name of dialect when --sql mode is set - *no DB connections # should occur, add this to env.py templates as a conditional* diff --git a/alembic/context.py b/alembic/context.py index 7b31cd94..4cf36284 100644 --- a/alembic/context.py +++ b/alembic/context.py @@ -32,10 +32,14 @@ class DefaultContext(object): transactional_ddl = False as_sql = False - def __init__(self, dialect, connection, fn, as_sql=False, + def __init__(self, dialect, script, connection, fn, as_sql=False, output_buffer=None, - transactional_ddl=None): + transactional_ddl=None, + starting_rev=None, + destination_rev=None, + tag=None): self.dialect = dialect + self.script = script if as_sql: self.connection = self._stdout_connection(connection) assert self.connection is not None @@ -49,14 +53,18 @@ class DefaultContext(object): self.output_buffer = output_buffer if transactional_ddl is not None: self.transactional_ddl = transactional_ddl + self._start_from_rev = starting_rev + self.destination_rev = destination_rev + self.tag = tag def _current_rev(self): if self.as_sql: - # TODO: no coverage here ! - # TODO: what if migrations table is needed on remote DB ?? - # need an option - raise Exception("revisions must be specified with --sql") + return self._start_from_rev else: + if self._start_from_rev: + raise util.CommandError( + "Can't specify current_rev to context " + "when using a database connection") _version.create(self.connection, checkfirst=True) return self.connection.scalar(_version.select()) @@ -87,8 +95,7 @@ class DefaultContext(object): current_rev = rev = False for change, prev_rev, rev in self._migrations_fn( - self._current_rev() - if not self.as_sql else None): + self._current_rev()): if current_rev is False: current_rev = prev_rev if self.as_sql and not current_rev: @@ -204,16 +211,18 @@ def _render_literal_bindparam(element, compiler, **kw): _context_opts = {} _context = None +_script = None -def opts(cfg, **kw): +def _opts(cfg, script, **kw): """Set up options that will be used by the :func:`.configure_connection` function. This basically sets some global variables. """ - global config + global config, _script _context_opts.update(kw) + _script = script config = cfg def requires_connection(): @@ -223,12 +232,45 @@ def requires_connection(): """ return not _context_opts.get('as_sql', False) +def get_head_revision(): + """Return the value of the 'head' revision.""" + rev = _script._get_rev('head') + if rev is not None: + return rev.revision + else: + return None + +def get_starting_revision_argument(): + """Return the 'starting revision' argument, + if the revision was passed as start:end. + + This is only usable in "offline" mode. + + """ + return get_context()._start_from_rev + +def get_revision_argument(): + """Get the 'destination' revision argument. + + This will be the target rev number. 'head' + is translated into the actual version number + as is 'base' which is translated to None. + + """ + return get_context().destination_rev + +def get_tag_argument(): + """Return the value passed for the ``--tag`` argument, if any.""" + return get_context().tag + def configure( connection=None, url=None, dialect_name=None, transactional_ddl=None, - output_buffer=None + output_buffer=None, + starting_rev=None, + tag=None ): """Configure the migration environment. @@ -259,6 +301,7 @@ def configure( :param output_buffer: a file-like object that will be used for textual output when the ``--sql`` option is used to generate SQL scripts. Defaults to ``sys.stdout`` it not passed here. + """ if connection: @@ -275,11 +318,17 @@ def configure( global _context from alembic.ddl import base opts = _context_opts.copy() - opts.setdefault("transactional_ddl", transactional_ddl) - opts.setdefault("output_buffer", output_buffer) + if transactional_ddl is not None: + opts["transactional_ddl"] = transactional_ddl + if output_buffer is not None: + opts["output_buffer"] = output_buffer + if starting_rev: + opts['starting_rev'] = starting_rev + if tag: + opts['tag'] = tag _context = _context_impls.get( dialect.name, - DefaultContext)(dialect, connection, **opts) + DefaultContext)(dialect, _script, connection, **opts) def configure_connection(connection): """Deprecated; use :func:`alembic.context.configure`.""" diff --git a/alembic/script.py b/alembic/script.py index d7a0856e..64bc2269 100644 --- a/alembic/script.py +++ b/alembic/script.py @@ -69,32 +69,15 @@ class ScriptDirectory(object): if script is None and lower is not None: raise util.CommandError("Couldn't find revision %s" % downrev) - # TODO: call range_ok -> as_sql and do as_sql validation - # here - range is required in as_sql mode, not allowed in - # non-as_sql mode. split into upgrade_to/upgrade_to_as_sql - def upgrade_from(self, range_ok, destination, current_rev): - if destination is not None and ':' in destination: - if not range_ok: - raise util.CommandError("Range revision not allowed") - revs = self._revs(*reversed(destination.split(':', 2))) - else: - revs = self._revs(destination, current_rev) + def upgrade_from(self, destination, current_rev): + revs = self._revs(destination, current_rev) return [ (script.module.upgrade, script.down_revision, script.revision) for script in reversed(list(revs)) ] - # TODO: call range_ok -> as_sql and do as_sql validation - # here - range is required in as_sql mode, not allowed in - # non-as_sql mode. split into downgrade_to/downgrade_to_as_sql - def downgrade_to(self, range_ok, destination, current_rev): - if destination is not None and ':' in destination: - if not range_ok: - raise util.CommandError("Range revision not allowed") - revs = self._revs(*destination.split(':', 2)) - else: - revs = self._revs(current_rev, destination) - + def downgrade_to(self, destination, current_rev): + revs = self._revs(current_rev, destination) return [ (script.module.downgrade, script.revision, script.down_revision) for script in revs diff --git a/alembic/templates/generic/env.py b/alembic/templates/generic/env.py index e10e682c..ac64bee7 100644 --- a/alembic/templates/generic/env.py +++ b/alembic/templates/generic/env.py @@ -1,16 +1,33 @@ from alembic import context from sqlalchemy import engine_from_config from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. config = context.config +# Interpret the config file for Pyhton logging. +# This line sets up loggers basically. fileConfig(config.config_file_name) +# Produce a SQLAlchemy engine using the key/values +# within the "alembic" section of the documentation, +# other otherwise what config_ini_section points to. engine = engine_from_config( - config.get_section('alembic'), prefix='sqlalchemy.') + config.get_section(config.config_ini_section), prefix='sqlalchemy.') + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +# if we're running in --sql mode, do everything connectionless. if not context.requires_connection(): context.configure(dialect_name=engine.name) context.run_migrations() + +# otherwise we need to make a connection. else: connection = engine.connect() context.configure(connection=connection, dialect_name=engine.name) diff --git a/alembic/templates/multidb/env.py b/alembic/templates/multidb/env.py index a46038ff..3acb27cf 100644 --- a/alembic/templates/multidb/env.py +++ b/alembic/templates/multidb/env.py @@ -3,6 +3,7 @@ USE_TWOPHASE = False from alembic import options, context from sqlalchemy import engine_from_config import re +import sys import logging logging.fileConfig(options.config_file) @@ -18,8 +19,12 @@ for name in re.split(r',\s*', db_names): if not context.requires_connection(): for name, rec in engines.items(): + # Write output to individual per-engine files. + file_ = "%s.sql" % name + sys.stderr.write("Writing output to %s\n" % file_) context.configure( - dialect_name=rec['engine'].name + dialect_name=rec['engine'].name, + output_buffer=file(file_, 'w') ) context.run_migrations(engine=name) else: diff --git a/alembic/templates/pylons/env.py b/alembic/templates/pylons/env.py index 4e9212cb..e202cea5 100644 --- a/alembic/templates/pylons/env.py +++ b/alembic/templates/pylons/env.py @@ -24,7 +24,8 @@ except: meta = __import__("%s.model.meta" % config['pylons.package']).model.meta if not context.requires_connection(): - context.configure(dialect_name=meta.engine.name) + context.configure( + dialect_name=meta.engine.name) context.run_migrations() else: connection = meta.engine.connect() diff --git a/docs/build/tutorial.rst b/docs/build/tutorial.rst index f801c644..af84b3af 100644 --- a/docs/build/tutorial.rst +++ b/docs/build/tutorial.rst @@ -121,7 +121,7 @@ The file generated with the "generic" configuration looks like:: # A generic, single database configuration. [alembic] - script_location = alembic + script_location = %(here)s/alembic sqlalchemy.url = driver://user:pass@localhost/dbname # Logging configuration @@ -159,6 +159,11 @@ The file generated with the "generic" configuration looks like:: format = %(levelname)-5.5s [%(name)s] %(message)s datefmt = %H:%M:%S +The file is read using Python's :class:`ConfigParser.ConfigParser` object, installing +the variable ``here`` as a substitution variable. This can be used to produce absolute +pathnames to directories and files, as we do above with the path to the Alembic +script location. + This file contains the following features: * ``[alembic]`` - this is the section read by Alembic to determine configuration. Alembic @@ -456,7 +461,8 @@ Generating SQL Scripts ====================== A major capability of Alembic is to generate migrations as SQL scripts, instead of running -them against the database. This is a critical feature when working in large organizations +them against the database - this is also referred to as "offline" mode. +This is a critical feature when working in large organizations where access to DDL is restricted, and SQL scripts must be handed off to DBAs. Alembic makes this easy via the ``--sql`` option passed to any ``upgrade`` or ``downgrade`` command. We can, for example, generate a script that revises up to rev ``ae1027a6acf``:: @@ -487,21 +493,37 @@ can, for example, generate a script that revises up to rev ``ae1027a6acf``:: While the logging configuration dumped to standard error, the actual script was dumped to standard output - -so typically we'd be using output redirection to generate a script:: +so in the absence of further configuration (described later in this section), we'd at first be using output +redirection to generate a script:: $ alembic upgrade ae1027a6acf --sql > migration.sql -Generating on a Range ---------------------- +Getting the Start Version +-------------------------- + +Notice that our migration script started at the base - this is the default when using offline +mode, as no database connection is present and there's no ``alembic_version`` table to read from. -Notice that our migration script started at the base - this is the default when using the ``--sql`` -operation, which does not otherwise make usage of a database connection, so does not retrieve -any starting version. We usually will want -to specify a start/end version. This is allowed when using the ``--sql`` option only -using the ``start:end`` syntax:: +One way to provide a starting version in offline mode is to provide a range to the command line. +This is accomplished by providing the "version" in ``start:end`` syntax:: $ alembic upgrade 1975ea83b712:ae1027a6acf --sql > migration.sql +The ``start:end`` syntax is only allowed in offline mode; in "online" mode, the ``alembic_version`` +table is always used to get at the current version. + +It's also possible to have the ``env.py`` script retrieve the "last" version from +the local environment, such as from a local file. A scheme like this would basically +treat a local file in the same way ``alembic_version`` works:: + + if not context.requires_connection(): + version_file = os.path.join(os.path.dirname(config.config_file_name), "version.txt")) + current_version = file_(version_file).read() + context.configure(dialect_name=engine.name, current_version=current_version) + start, end = context.run_migrations() + if end: + file_(version_file, 'w').write(end) + Writing Migration Scripts to Support Script Generation ------------------------------------------------------ diff --git a/tests/__init__.py b/tests/__init__.py index 1fec30c9..8788f1f3 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,13 +3,15 @@ import shutil import os import itertools from sqlalchemy import create_engine, text -from alembic import context +from alembic import context, util import re +from alembic.script import ScriptDirectory from alembic.context import _context_impls from alembic import ddl import StringIO staging_directory = os.path.join(os.path.dirname(__file__), 'scratch') +files_directory = os.path.join(os.path.dirname(__file__), 'files') _dialects = {} def _get_dialect(name): @@ -86,6 +88,7 @@ def _op_fixture(dialect='default', as_sql=False): # TODO: make this more flexible about # whitespace and such eq_(self.assertion, list(sql)) + _context_impls[dialect] = _base return ctx(dialect, as_sql) def _sqlite_testing_config(): @@ -122,6 +125,22 @@ datefmt = %%H:%%M:%%S """ % (dir_, dir_)) return cfg +def _env_file_fixture(txt): + dir_ = os.path.join(staging_directory, 'scripts') + txt = """ +from alembic import context + +config = context.config +""" + txt + + path = os.path.join(dir_, "env.py") + pyc_path = util.pyc_file_from_path(path) + if os.access(pyc_path, os.F_OK): + os.unlink(pyc_path) + + file(path, 'w').write(txt) + + def _no_sql_testing_config(): """use a postgresql url with no host so that connections guaranteed to fail""" cfg = _testing_config() @@ -174,3 +193,53 @@ def staging_env(create=True): def clear_staging_env(): shutil.rmtree(staging_directory, True) + + +def three_rev_fixture(cfg): + a = util.rev_id() + b = util.rev_id() + c = util.rev_id() + + script = ScriptDirectory.from_config(cfg) + script.generate_rev(a, None) + script.write(a, """ +down_revision = None + +from alembic.op import * + +def upgrade(): + execute("CREATE STEP 1") + +def downgrade(): + execute("DROP STEP 1") + +""") + + script.generate_rev(b, None) + script.write(b, """ +down_revision = '%s' + +from alembic.op import * + +def upgrade(): + execute("CREATE STEP 2") + +def downgrade(): + execute("DROP STEP 2") + +""" % a) + + script.generate_rev(c, None) + script.write(c, """ +down_revision = '%s' + +from alembic.op import * + +def upgrade(): + execute("CREATE STEP 3") + +def downgrade(): + execute("DROP STEP 3") + +""" % b) + return a, b, c \ No newline at end of file diff --git a/tests/test_offline_environment.py b/tests/test_offline_environment.py new file mode 100644 index 00000000..c8847d5f --- /dev/null +++ b/tests/test_offline_environment.py @@ -0,0 +1,72 @@ +from tests import clear_staging_env, staging_env, \ + _no_sql_testing_config, sqlite_db, eq_, ne_, \ + capture_context_buffer, three_rev_fixture, _env_file_fixture +from alembic import command, util + +def setup(): + global cfg, env + env = staging_env() + cfg = _no_sql_testing_config() + + global a, b, c + a, b, c = three_rev_fixture(cfg) + +def teardown(): + clear_staging_env() + +def test_not_requires_connection(): + _env_file_fixture(""" +assert not context.requires_connection() +""") + command.upgrade(cfg, a, sql=True) + command.downgrade(cfg, a, sql=True) + +def test_requires_connection(): + _env_file_fixture(""" +assert context.requires_connection() +""") + command.upgrade(cfg, a) + command.downgrade(cfg, a) + + +def test_starting_rev(): + _env_file_fixture(""" +context.configure(dialect_name='sqlite', starting_rev='x') +assert context.get_starting_revision_argument() == 'x' +""") + command.upgrade(cfg, a, sql=True) + command.downgrade(cfg, a, sql=True) + + +def test_destination_rev(): + _env_file_fixture(""" +context.configure(dialect_name='sqlite') +assert context.get_revision_argument() == '%s' +""" % b) + command.upgrade(cfg, b, sql=True) + command.downgrade(cfg, b, sql=True) + + +def test_head_rev(): + _env_file_fixture(""" +context.configure(dialect_name='sqlite') +assert context.get_head_revision() == '%s' +""" % c) + command.upgrade(cfg, b, sql=True) + command.downgrade(cfg, b, sql=True) + +def test_tag_cmd_arg(): + _env_file_fixture(""" +context.configure(dialect_name='sqlite') +assert context.get_tag_argument() == 'hi' +""") + command.upgrade(cfg, b, sql=True, tag='hi') + command.downgrade(cfg, b, sql=True, tag='hi') + +def test_tag_cfg_arg(): + _env_file_fixture(""" +context.configure(dialect_name='sqlite', tag='there') +assert context.get_tag_argument() == 'there' +""") + command.upgrade(cfg, b, sql=True, tag='hi') + command.downgrade(cfg, b, sql=True, tag='hi') diff --git a/tests/test_revision_paths.py b/tests/test_revision_paths.py index 1320a33c..c6e0ea67 100644 --- a/tests/test_revision_paths.py +++ b/tests/test_revision_paths.py @@ -19,7 +19,7 @@ def teardown(): def test_upgrade_path(): eq_( - env.upgrade_from(False, e.revision, c.revision), + env.upgrade_from(e.revision, c.revision), [ (d.module.upgrade, c.revision, d.revision), (e.module.upgrade, d.revision, e.revision), @@ -27,7 +27,7 @@ def test_upgrade_path(): ) eq_( - env.upgrade_from(False, c.revision, None), + env.upgrade_from(c.revision, None), [ (a.module.upgrade, None, a.revision), (b.module.upgrade, a.revision, b.revision), @@ -38,7 +38,7 @@ def test_upgrade_path(): def test_downgrade_path(): eq_( - env.downgrade_to(False, c.revision, e.revision), + env.downgrade_to(c.revision, e.revision), [ (e.module.downgrade, e.revision, e.down_revision), (d.module.downgrade, d.revision, d.down_revision), @@ -46,7 +46,7 @@ def test_downgrade_path(): ) eq_( - env.downgrade_to(False, None, c.revision), + env.downgrade_to(None, c.revision), [ (c.module.downgrade, c.revision, c.down_revision), (b.module.downgrade, b.revision, b.down_revision), diff --git a/tests/test_sql_script.py b/tests/test_sql_script.py index 5fa7fe51..54538a10 100644 --- a/tests/test_sql_script.py +++ b/tests/test_sql_script.py @@ -1,6 +1,5 @@ -from tests import clear_staging_env, staging_env, _no_sql_testing_config, sqlite_db, eq_, ne_, capture_context_buffer +from tests import clear_staging_env, staging_env, _no_sql_testing_config, sqlite_db, eq_, ne_, capture_context_buffer, three_rev_fixture from alembic import command, util -from alembic.script import ScriptDirectory def setup(): global cfg, env @@ -8,52 +7,7 @@ def setup(): cfg = _no_sql_testing_config() global a, b, c - a = util.rev_id() - b = util.rev_id() - c = util.rev_id() - - script = ScriptDirectory.from_config(cfg) - script.generate_rev(a, None) - script.write(a, """ -down_revision = None - -from alembic.op import * - -def upgrade(): - execute("CREATE STEP 1") - -def downgrade(): - execute("DROP STEP 1") - -""") - - script.generate_rev(b, None) - script.write(b, """ -down_revision = '%s' - -from alembic.op import * - -def upgrade(): - execute("CREATE STEP 2") - -def downgrade(): - execute("DROP STEP 2") - -""" % a) - - script.generate_rev(c, None) - script.write(c, """ -down_revision = '%s' - -from alembic.op import * - -def upgrade(): - execute("CREATE STEP 3") - -def downgrade(): - execute("DROP STEP 3") - -""" % b) + a, b, c = three_rev_fixture(cfg) def teardown(): clear_staging_env()