From: Mike Bayer Date: Sun, 25 Apr 2010 03:51:21 +0000 (-0400) Subject: - figuring out script format X-Git-Tag: rel_0_1_0~114 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=72bf3a5417fd968496e01c48a8c7d1bfa6b40676;p=thirdparty%2Fsqlalchemy%2Falembic.git - figuring out script format - figuring out operation system --- diff --git a/.hgignore b/.hgignore new file mode 100644 index 00000000..45f1fab9 --- /dev/null +++ b/.hgignore @@ -0,0 +1,8 @@ +syntax:regexp +^build/ +^dist/ +^doc/build/output +.pyc$ +.orig$ +.egg-info + diff --git a/alembic/__init__.py b/alembic/__init__.py index 25b4cfe1..304c26b2 100644 --- a/alembic/__init__.py +++ b/alembic/__init__.py @@ -1,4 +1,4 @@ from command import main -__version__ = "0.1" +__version__ = '0.1alpha' diff --git a/alembic/command.py b/alembic/command.py index e8fa4d37..ab40617d 100644 --- a/alembic/command.py +++ b/alembic/command.py @@ -1,23 +1,26 @@ from alembic.script import Script -def main(options, file_config, command): +def main(options, command): raise NotImplementedError("yeah yeah nothing here yet") -def init(options, file_config): +def init(options): """Initialize a new scripts directory.""" - script = Script(options, file_config) + script = ScriptDirectory(options) script.init() -def upgrade(options, file_config): +def upgrade(options): """Upgrade to the latest version.""" - script = Script(options, file_config) + script = ScriptDirectory(options) + + # ... def revert(options, file_config): """Revert to a specific previous version.""" - script = Script(options, file_config) + script = ScriptDirectory(options) + # ... diff --git a/alembic/context.py b/alembic/context.py new file mode 100644 index 00000000..27771daa --- /dev/null +++ b/alembic/context.py @@ -0,0 +1,31 @@ +from alembic.ddl import base + +class ContextMeta(type): + def __init__(cls, classname, bases, dict_): + newtype = type.__init__(cls, classname, bases, dict_) + if '__dialect__' in dict_: + _context_impls[dict_['__dialect__']] = newtype + return newtype + +_context_impls = {} + +class DefaultContext(object): + __metaclass__ = ContextMeta + + def __init__(self, options, connection): + self.options = options + self.connection = connection + + def alter_column(self, table_name, column_name, + nullable=NO_VALUE, + server_default=NO_VALUE, + name=NO_VALUE, + type=NO_VALUE + ): + + if nullable is not NO_VALUE: + base.ColumnNullable(table_name, column_name, nullable) + if server_default is not NO_VALUE: + base.ColumnDefault(table_name, column_name, server_default) + + # ... etc \ No newline at end of file diff --git a/alembic/ddl/mysql.py b/alembic/ddl/mysql.py new file mode 100644 index 00000000..0b60bf2a --- /dev/null +++ b/alembic/ddl/mysql.py @@ -0,0 +1,5 @@ +from alembic.context import DefaultContext + +class MySQLContext(DefaultContext): + __dialect__ = 'mysql' + diff --git a/alembic/ddl/op.py b/alembic/ddl/op.py deleted file mode 100644 index f0f96b10..00000000 --- a/alembic/ddl/op.py +++ /dev/null @@ -1,15 +0,0 @@ - -class DefaultContext(object): - def alter_column(self, table_name, column_name, - nullable=NO_VALUE, - server_default=NO_VALUE, - name=NO_VALUE, - type=NO_VALUE - ): - - if nullable is not NO_VALUE: - ColumnNullable(table_name, column_name, nullable) - if server_default is not NO_VALUE: - ColumnDefault(table_name, column_name, server_default) - - # ... etc \ No newline at end of file diff --git a/alembic/ddl/postgresql.py b/alembic/ddl/postgresql.py index e69de29b..894a3026 100644 --- a/alembic/ddl/postgresql.py +++ b/alembic/ddl/postgresql.py @@ -0,0 +1,6 @@ +from alembic.context import DefaultContext + +class PostgresqlContext(DefaultContext): + __dialect__ = 'postgresql' + + \ No newline at end of file diff --git a/alembic/options.py b/alembic/options.py index 0130b5e9..afc26c24 100644 --- a/alembic/options.py +++ b/alembic/options.py @@ -1,20 +1,20 @@ def get_option_parser(): parser = OptionParser("usage: %prog [options] ") - parser.add_option("-d", "--dir", type="string", action="store", help="Location of script directory.") - + return parser class Options(object): - def __init__(self, options): - self.options = options + def __init__(self, cmd_line_options): + self.cmd_line_options = cmd_line_options self.file_config = ConfigParser.ConfigParser() # TODO: cfg file can come from options self.file_config.read(['alembic.cfg']) + + def get_section(self, name): + return dict(self.file_config.items(name)) def get_main_option(self, name, default=None): - if getattr(self.options, name): - return getattr(self.options, name) - elif self.file_config.get('alembic', name): + if self.file_config.get('alembic', name): return self.file_config.get('alembic', name) else: return default diff --git a/alembic/script.py b/alembic/script.py index 79ce1ba5..1bdd02f4 100644 --- a/alembic/script.py +++ b/alembic/script.py @@ -6,22 +6,9 @@ class ScriptDirectory(object): @classmethod def from_options(cls, options, file_config): - if options.dir: - d = options.dir - elif file_config.get('alembic', 'dir'): - d = file_config.get('alembic', 'dir') - else: - d = os.path.join(os.path.curdir, "alembic_scripts") - return Script(d) - - + return Script(file_config.get_main_option('script_location')) + def init(self): if not os.access(self.dir, os.F_OK): os.makedirs(self.dir) - f = open(os.path.join(self.dir, "env.py"), 'w') - f.write( - "def startup(options, file_config):" - " pass # TOOD" - ) - f.close() - + # copy files... diff --git a/sample_notes.txt b/sample_notes.txt index d17cbfc1..e6b87124 100644 --- a/sample_notes.txt +++ b/sample_notes.txt @@ -3,7 +3,30 @@ # commands: +alembic list-templates + Available templates: + + generic - plain migration template + pylons - uses Pylons .ini file and environment + multidb - illustrates multi-database usage + alembic init ./scripts + --template argument is required. + + Available templates: + + generic - plain migration template + pylons - uses Pylons .ini file and environment + multidb - illustrates multi-database usage + +alembic init --template=generic ./scripts + using template 'generic'.... + creating script dir './scripts'.... + copying 'env.py'... + copying 'script.py.mako'... + generating 'alembic.ini'... + done ! edit 'sqlalchemy.url' in alembic.ini + alembic revision -m "do something else" alembic upgrade alembic revert -r jQq57 @@ -13,69 +36,13 @@ alembic branches # with pylons - use paster commands ? - scripts directory looks like: - ./scripts/ env.py - script_template.py + script.py.mako lfh56_do_this.py jQq57_do_that.py Pzz19_do_something_else.py - -# env.py looks like: -# ------------------------------------ - -from alembic import options, context - -import logging -logging.fileConfig(options.config_file) - -engine = create_engine(options.get_main_option('url')) - -connection = engine.connect() -context.configure_connection(connection) -trans = connection.begin() -try: - run_migrations() - trans.commit() -except: - trans.rollback() - - -# a pylons env.py looks like: -# -------------------------------------- - -from alembic import options, context - -from name_of_project.config import environment -environment.setup_app(whatever) - -engine = environment.engine # or meta.engine, whatever it has to be here - -connection = engine.connect() -context.configure_connection(connection) -trans = connection.begin() -try: - run_migrations() - trans.commit() -except: - trans.rollback() - -# script template -# ----------------------------------------- -from alembic.op import * - -def upgrade_%(up_revision)s(): - pass - -def downgrade_%(down_revision)s(): - pass - - - - - \ No newline at end of file diff --git a/setup.py b/setup.py index 54ee5612..30bc17ad 100644 --- a/setup.py +++ b/setup.py @@ -6,27 +6,52 @@ v = open(os.path.join(os.path.dirname(__file__), 'alembic', '__init__.py')) VERSION = re.compile(r".*__version__ = '(.*?)'", re.S).match(v.read()).group(1) v.close() + +def datafiles(): + out = [] + for root, dirs, files in os.walk('./templates'): + if files: + out.append((root, [os.path.join(root, f) for f in files])) + return out + setup(name='alembic', version=VERSION, - description="A database migration tool for SQLAlchemy." + description="A database migration tool for SQLAlchemy.", long_description="""\ -alembic allows the creation of script files which specify a particular revision of a database, -and its movements to other revisions. The scripting system executes within the context -of a particular connection and transactional configuration, and encourages the usage of -SQLAlchemy DDLElement constructs and table reflection in order to execute changes to -schemas. - -The current goal of the tool is to allow explicit but minimalistic scripting -between any two states, with only a simplistic model of dependency traversal -working in the background to execute a series of steps. - -The author is well aware that the goal of "minimal and simplistic" is how -tools both underpowered and massively bloated begin. It is hoped that -Alembic's basic idea is useful enough that the tool can remain straightforward -without the need for vast tracts of complexity to be added, but that -remains to be seen. +Alembic is an open ended migrations tool. +Basic operation involves the creation of script files, +each representing a version transition for one or more databases. +The scripts execute within the context of a particular connection +and transactional configuration that is locally configurable. +Key goals of Alembic are: + * extremely flexible and obvious configuration, including the + capability to deal with multiple database setups, both vertical + and horizontally partioned patterns + * complete control over the engine/transactional environment + in which migrations run, with an emphasis on all migrations running + under a single transaction per-engine if supported by the + underlying database, as well as using two-phase commit if + running with multiple databases. + * The ability to generate any set of migration scripts as textual + SQL files. + * rudimental capability to deal with source code branches, + by organizing migration scripts based on dependency references, + and providing a "splice" command to bridge two branches. + * allowing an absolute minimum of typing, both to run commands + as well as to create new migrations. Simple migration + commands on existing schema constructs use only strings and + flags, not requiring the usage of SQLAlchemy metadata objects. + Columns can be added without the need for Table/MetaData + objects. Engines and connections need not be referenced + in script files. + * Old migration files can be deleted if those older versions + are no longer needed. + * The ability to integrate seamlessly and simply with frameworks + such as Pylons, using the framework's SQLAlchemy environment + to keep database connection configuration centralized. + """, classifiers=[ 'Development Status :: 3 - Alpha', @@ -42,11 +67,13 @@ remains to be seen. license='MIT', packages=find_packages('.', exclude=['examples*', 'test*']), scripts=['scripts/alembic'], + data_files=datafiles(), tests_require = ['nose >= 0.11'], test_suite = "nose.collector", zip_safe=False, install_requires=[ 'SQLAlchemy>=0.6.0', + 'Mako' ], entry_points=""" """, diff --git a/templates/generic/alembic.ini.mako b/templates/generic/alembic.ini.mako new file mode 100644 index 00000000..1b3b7c4c --- /dev/null +++ b/templates/generic/alembic.ini.mako @@ -0,0 +1,41 @@ +# A generic, single database configuration. + +[alembic] +script_location = ${script_location} +sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/templates/generic/env.py b/templates/generic/env.py new file mode 100644 index 00000000..e11c4a6c --- /dev/null +++ b/templates/generic/env.py @@ -0,0 +1,16 @@ +from alembic import options, context +from sqlalchemy import engine_from_config +import logging + +logging.fileConfig(options.config_file) + +engine = engine_from_config(options.get_section('alembic'), prefix='sqlalchemy.') + +connection = engine.connect() +context.configure_connection(connection) +trans = connection.begin() +try: + run_migrations() + trans.commit() +except: + trans.rollback() diff --git a/templates/generic/script.py.mako b/templates/generic/script.py.mako new file mode 100644 index 00000000..4181f3a9 --- /dev/null +++ b/templates/generic/script.py.mako @@ -0,0 +1,7 @@ +from alembic.op import * + +def upgrade_${up_revision}(): + pass + +def downgrade_${down_revision}(): + pass diff --git a/templates/multidb/alembic.ini.mako b/templates/multidb/alembic.ini.mako new file mode 100644 index 00000000..08c27b0d --- /dev/null +++ b/templates/multidb/alembic.ini.mako @@ -0,0 +1,47 @@ +# a multi-database configuration. + +[alembic] +script_location = ${script_location} +databases = engine1, engine2 + +[engine1] +sqlalchemy.url = driver://user:pass@localhost/dbname + +[engine2] +sqlalchemy.url = driver://user:pass@localhost/dbname2 + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/templates/multidb/env.py b/templates/multidb/env.py new file mode 100644 index 00000000..22746cb2 --- /dev/null +++ b/templates/multidb/env.py @@ -0,0 +1,37 @@ +USE_TWOPHASE = False + +from alembic import options, context +from sqlalchemy import engine_from_config +import re + +import logging +logging.fileConfig(options.config_file) + +db_names = options.get_main_option('databases') + +engines = {} +for name in re.split(r',\s*', db_names): + engines[name] = rec = {} + rec['engine'] = engine = \ + engine_from_config(options.get_section(name), prefix='sqlalchemy.') + rec['connection'] = conn = engine.connect() + + if USE_TWOPHASE: + rec['transaction'] = conn.begin_twophase() + else: + rec['transaction'] = conn.begin() + +try: + for name, rec in engines.items(): + context.configure_connection(rec['connection']) + run_migrations(engine=name) + + if USE_TWOPHASE: + for rec in engines.values(): + rec['transaction'].prepare() + + for rec in engines.values(): + rec['transaction'].commit() +except: + for rec in engines.values(): + rec['transaction'].rollback() diff --git a/templates/multidb/script.py.mako b/templates/multidb/script.py.mako new file mode 100644 index 00000000..ee220144 --- /dev/null +++ b/templates/multidb/script.py.mako @@ -0,0 +1,9 @@ +from alembic.op import * + +% for engine in engines: +def upgrade_${engine}_${up_revision}(): + pass + +def downgrade_${engine}_${down_revision}(): + pass +% endfor \ No newline at end of file diff --git a/templates/pylons/alembic.ini.mako b/templates/pylons/alembic.ini.mako new file mode 100644 index 00000000..19bcb415 --- /dev/null +++ b/templates/pylons/alembic.ini.mako @@ -0,0 +1,7 @@ +# a Pylons configuration. + +[alembic] +script_location = ${script_location} +pylons_config_file = ./development.ini + +# that's it ! \ No newline at end of file diff --git a/templates/pylons/env.py b/templates/pylons/env.py new file mode 100644 index 00000000..8afa1087 --- /dev/null +++ b/templates/pylons/env.py @@ -0,0 +1,26 @@ +"""Pylons bootstrap environment. + +Place 'pylons_config_file' into alembic.ini, and the application will +be loaded from there. + +""" +from alembic import options, context +from paste.deploy import loadapp +from pylons import config +import logging + +config_file = options.get_main_option('pylons_config_file') +logging.fileConfig(config_file) +wsgi_app = loadapp('config:%s' % config_file, relative_to='.') + +# customize this section for non-standard engine configurations. +meta = __import__("%s.model.meta" % config['pylons.package']).model.meta + +connection = meta.engine.connect() +context.configure_connection(connection) +trans = connection.begin() +try: + run_migrations() + trans.commit() +except: + trans.rollback() diff --git a/templates/pylons/script.py.mako b/templates/pylons/script.py.mako new file mode 100644 index 00000000..4181f3a9 --- /dev/null +++ b/templates/pylons/script.py.mako @@ -0,0 +1,7 @@ +from alembic.op import * + +def upgrade_${up_revision}(): + pass + +def downgrade_${down_revision}(): + pass