From: Mike Bayer Date: Sun, 30 Sep 2012 18:57:11 +0000 (-0400) Subject: - [feature] Support for tables in alternate schemas X-Git-Tag: rel_0_4_0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a654c35bd810f416d393b7102118b85cec787b1a;p=thirdparty%2Fsqlalchemy%2Falembic.git - [feature] Support for tables in alternate schemas has been added fully to all operations, as well as to the autogenerate feature. When using autogenerate, specifying the flag include_schemas=True to Environment.configure() will also cause autogenerate to scan all schemas located by Inspector.get_schema_names(), which is supported by *some* (but not all) SQLAlchemy dialects including Postgresql. *Enormous* thanks to Bruno Binet for a huge effort in implementing as well as writing tests. #33. --- diff --git a/CHANGES b/CHANGES index 5e1265f0..c74d5ae4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,16 @@ 0.4.0 ===== +- [feature] Support for tables in alternate schemas + has been added fully to all operations, as well as to + the autogenerate feature. When using autogenerate, + specifying the flag include_schemas=True to + Environment.configure() will also cause autogenerate + to scan all schemas located by Inspector.get_schema_names(), + which is supported by *some* (but not all) + SQLAlchemy dialects including Postgresql. + *Enormous* thanks to Bruno Binet for a huge effort + in implementing as well as writing tests. #33. + - [feature] The command line runner has been organized into a reusable CommandLine object, so that other front-ends can re-use the argument parsing built diff --git a/alembic/autogenerate.py b/alembic/autogenerate.py index ebb8396b..b6af35cf 100644 --- a/alembic/autogenerate.py +++ b/alembic/autogenerate.py @@ -105,10 +105,12 @@ def compare_metadata(context, metadata): # top level def _produce_migration_diffs(context, template_args, - imports, include_symbol=None): + imports, include_symbol=None, + include_schemas=False): opts = context.opts metadata = opts['target_metadata'] include_symbol = opts.get('include_symbol', include_symbol) + include_schemas = opts.get('include_schemas', include_schemas) if metadata is None: raise util.CommandError( @@ -121,7 +123,8 @@ def _produce_migration_diffs(context, template_args, diffs = [] _produce_net_changes(connection, metadata, diffs, - autogen_context, include_symbol) + autogen_context, include_symbol, + include_schemas) template_args[opts['upgrade_token']] = \ _indent(_produce_upgrade_commands(diffs, autogen_context)) template_args[opts['downgrade_token']] = \ @@ -150,15 +153,22 @@ def _indent(text): # walk structures def _produce_net_changes(connection, metadata, diffs, autogen_context, - include_symbol=None): + include_symbol=None, + include_schemas=False): inspector = Inspector.from_engine(connection) # TODO: not hardcode alembic_version here ? conn_table_names = set() - schemas = inspector.get_schema_names() or [None] + if include_schemas: + schemas = set(inspector.get_schema_names()) + # replace default schema name with None + schemas.discard("information_schema") + # replace the "default" schema with None + schemas.add(None) + schemas.discard(connection.dialect.default_schema_name) + else: + schemas = [None] + for s in schemas: - if s == 'information_schema': - # ignore postgres own information_schema - continue tables = set(inspector.get_table_names(schema=s)).\ difference(['alembic_version']) conn_table_names.update(zip([s] * len(tables), tables)) @@ -169,10 +179,10 @@ def _produce_net_changes(connection, metadata, diffs, autogen_context, if include_symbol: conn_table_names = set((s, name) for s, name in conn_table_names - if include_symbol(name, schema=s)) + if include_symbol(name, s)) metadata_table_names = OrderedSet((s, name) for s, name in metadata_table_names - if include_symbol(name, schema=s)) + if include_symbol(name, s)) _compare_tables(conn_table_names, metadata_table_names, inspector, metadata, diffs, autogen_context) diff --git a/alembic/environment.py b/alembic/environment.py index 82bd7610..7b0d16d0 100644 --- a/alembic/environment.py +++ b/alembic/environment.py @@ -209,6 +209,7 @@ class EnvironmentContext(object): template_args=None, target_metadata=None, include_symbol=None, + include_schemas=False, compare_type=False, compare_server_default=False, upgrade_token="upgrades", @@ -357,11 +358,11 @@ class EnvironmentContext(object): the two defaults on the database side to compare for equivalence. :param include_symbol: A callable function which, given a table name - and optional schema name, returns ``True`` or ``False``, indicating + and schema name (may be ``None``), returns ``True`` or ``False``, indicating if the given table should be considered in the autogenerate sweep. E.g.:: - def include_symbol(tablename, schema=None): + def include_symbol(tablename, schema): return tablename not in ("skip_table_one", "skip_table_two") context.configure( @@ -369,8 +370,23 @@ class EnvironmentContext(object): include_symbol = include_symbol ) + To limit autogenerate to a certain set of schemas when using the + ``include_schemas`` option:: + + def include_symbol(tablename, schema): + return schema in (None, "schema1", "schema2") + + context.configure( + # ... + include_schemas = True, + include_symbol = include_symbol + ) + .. versionadded:: 0.3.6 + .. versionchanged:: 0.4.0 the ``include_symbol`` callable must now + also accept a "schema" argument, which may be None. + :param upgrade_token: When autogenerate completes, the text of the candidate upgrade operations will be present in this template variable when ``script.py.mako`` is rendered. Defaults to @@ -395,6 +411,16 @@ class EnvironmentContext(object): will render them using the dialect module name, i.e. ``mssql.BIT()``, ``postgresql.UUID()``. + :param include_schemas: If True, autogenerate will scan across + all schemas located by the SQLAlchemy + :meth:`~sqlalchemy.engine.reflection.Inspector.get_schema_names` + method, and include all differences in tables found across all + those schemas. When using this option, you may want to also + use the ``include_symbol`` option to specify a callable which + can filter the tables/schemas that get included. + + .. versionadded :: 0.4.0 + Parameters specific to individual backends: :param mssql_batch_separator: The "batch separator" which will @@ -412,7 +438,7 @@ class EnvironmentContext(object): """ opts = self.context_opts if transactional_ddl is not None: - opts["transactional_ddl"] = transactional_ddl + opts["transactional_ddl"] = transactional_ddl if output_buffer is not None: opts["output_buffer"] = output_buffer elif self.config.output_buffer is not None: @@ -425,6 +451,7 @@ class EnvironmentContext(object): opts['template_args'].update(template_args) opts['target_metadata'] = target_metadata opts['include_symbol'] = include_symbol + opts['include_schemas'] = include_schemas opts['upgrade_token'] = upgrade_token opts['downgrade_token'] = downgrade_token opts['sqlalchemy_module_prefix'] = sqlalchemy_module_prefix diff --git a/alembic/operations.py b/alembic/operations.py index 01d29e95..0d012937 100644 --- a/alembic/operations.py +++ b/alembic/operations.py @@ -153,7 +153,7 @@ class Operations(object): :param old_table_name: old name. :param new_table_name: new name. - :param schema: Optional, name of schema to operate within. + :param schema: Optional schema name to operate within. """ self.impl.rename_table( @@ -240,7 +240,10 @@ class Operations(object): :param existing_autoincrement: Optional; the existing autoincrement of the column. Used for MySQL's system of altering a column that specifies ``AUTO_INCREMENT``. - :param schema: Optional, name of schema to operate within. + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 + """ compiler = self.impl.dialect.statement_compiler( @@ -326,7 +329,9 @@ class Operations(object): :param table_name: String name of the parent table. :param column: a :class:`sqlalchemy.schema.Column` object representing the new column. - :param schema: Optional, name of schema to operate within. + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 """ @@ -350,6 +355,10 @@ class Operations(object): :param table_name: name of table :param column_name: name of column + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 + :param mssql_drop_check: Optional boolean. When ``True``, on Microsoft SQL Server only, first drop the CHECK constraint on the column using a @@ -458,7 +467,9 @@ class Operations(object): issuing DDL for this constraint. :param initially: optional string. If set, emit INITIALLY when issuing DDL for this constraint. - :param schema: Optional schema name of the source table. + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 """ @@ -502,7 +513,9 @@ class Operations(object): issuing DDL for this constraint. :param initially: optional string. If set, emit INITIALLY when issuing DDL for this constraint. - :param schema: Optional schema name of the source table. + :param schema: Optional schema name to operate within. + + ..versionadded:: 0.4.0 """ self.impl.add_constraint( @@ -552,6 +565,7 @@ class Operations(object): ``after_create`` events when the table is being created. In particular, the Postgresql ENUM type will emit a CREATE TYPE within these events. + :param schema: Optional schema name to operate within. :param \**kw: Other keyword arguments are passed to the underlying :class:`.Table` object created for the command. @@ -570,6 +584,10 @@ class Operations(object): drop_table("accounts") :param name: Name of the table + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 + :param \**kw: Other keyword arguments are passed to the underlying :class:`.Table` object created for the command. @@ -591,7 +609,9 @@ class Operations(object): :param tablename: name of the owning table. :param columns: a list of string column names in the table. - :param schema: Optional, name of schema to operate within. + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 """ @@ -611,7 +631,9 @@ class Operations(object): :param name: name of the index. :param tablename: name of the owning table. Some backends such as Microsoft SQL Server require this. - :param schema: Optional, name of schema to operate within. + :param schema: Optional schema name to operate within. + + .. versionadded:: 0.4.0 """ # need a dummy column name here since SQLAlchemy @@ -628,10 +650,12 @@ class Operations(object): :param type: optional, required on MySQL. can be 'foreignkey', 'primary', 'unique', or 'check'. - .. versionadded:: 0.3.6 'primary' qualfier to enable - dropping of MySQL primary key constraints. + .. versionadded:: 0.3.6 'primary' qualfier to enable + dropping of MySQL primary key constraints. + + :param schema: Optional schema name to operate within. - :param schema: Optional, name of schema to operate within. + .. versionadded:: 0.4.0 """ t = self._table(tablename, schema=schema) diff --git a/tests/__init__.py b/tests/__init__.py index 1456806a..2b128362 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -52,7 +52,7 @@ def db_for_dialect(name): except ConfigParser.NoOptionError: raise SkipTest("No dialect %r in test.cfg" % name) try: - eng = create_engine(cfg, echo=True) + eng = create_engine(cfg) #, echo=True) except ImportError, er1: raise SkipTest("Can't import DBAPI: %s" % er1) try: diff --git a/tests/test_autogenerate.py b/tests/test_autogenerate.py index d9a0c7df..a20b1bde 100644 --- a/tests/test_autogenerate.py +++ b/tests/test_autogenerate.py @@ -90,7 +90,9 @@ def _model_four(): return m - +_default_include_symbol = lambda name, schema=None: name in ("parent", "child", + "user", "order", "item", + "address", "extra") class AutogenTest(object): @classmethod @@ -183,7 +185,7 @@ class ImplicitConstraintNoGenTest(AutogenTest, TestCase): template_args = {} autogenerate._produce_migration_diffs(self.context, template_args, set(), - include_symbol=lambda name: name in ('sometable', 'someothertable') + include_symbol=lambda name, schema=None: name in ('sometable', 'someothertable') ) eq_( re.sub(r"u'", "'", template_args['downgrades']), @@ -199,20 +201,102 @@ class ImplicitConstraintNoGenTest(AutogenTest, TestCase): ) +class AutogenCrossSchemaTest(AutogenTest, TestCase): + @classmethod + def _get_bind(cls): + cls.test_schema_name = "test_schema" + return db_for_dialect('postgresql') + + @classmethod + def _get_db_schema(cls): + m = MetaData() + Table('t1', m, + Column('x', Integer) + ) + Table('t2', m, + Column('y', Integer), + schema=cls.test_schema_name + ) + return m + + @classmethod + def _get_model_schema(cls): + m = MetaData() + Table('t3', m, + Column('q', Integer) + ) + Table('t4', m, + Column('z', Integer), + schema=cls.test_schema_name + ) + return m + + def test_default_schema_omitted_upgrade(self): + metadata = self.m2 + connection = self.context.bind + diffs = [] + autogenerate._produce_net_changes(connection, metadata, diffs, + self.autogen_context, + include_symbol=lambda n, s: n == 't3', + include_schemas=True + ) + eq_(diffs[0][0], "add_table") + eq_(diffs[0][1].schema, None) + + def test_alt_schema_included_upgrade(self): + metadata = self.m2 + connection = self.context.bind + diffs = [] + autogenerate._produce_net_changes(connection, metadata, diffs, + self.autogen_context, + include_symbol=lambda n, s: n == 't4', + include_schemas=True + ) + eq_(diffs[0][0], "add_table") + eq_(diffs[0][1].schema, self.test_schema_name) + + def test_default_schema_omitted_downgrade(self): + metadata = self.m2 + connection = self.context.bind + diffs = [] + autogenerate._produce_net_changes(connection, metadata, diffs, + self.autogen_context, + include_symbol=lambda n, s: n == 't1', + include_schemas=True + ) + eq_(diffs[0][0], "remove_table") + eq_(diffs[0][1].schema, None) + + def test_alt_schema_included_downgrade(self): + metadata = self.m2 + connection = self.context.bind + diffs = [] + autogenerate._produce_net_changes(connection, metadata, diffs, + self.autogen_context, + include_symbol=lambda n, s: n == 't2', + include_schemas=True + ) + eq_(diffs[0][0], "remove_table") + eq_(diffs[0][1].schema, self.test_schema_name) + + + class AutogenerateDiffTestWSchema(AutogenTest, TestCase): @classmethod def _get_bind(cls): + cls.test_schema_name = "test_schema" return db_for_dialect('postgresql') @classmethod def _get_db_schema(cls): - return _model_one(schema='foo') + return _model_one(schema=cls.test_schema_name) @classmethod def _get_model_schema(cls): - return _model_two(schema='foo') + return _model_two(schema=cls.test_schema_name) + def test_diffs(self): """test generation of diff rules""" @@ -221,28 +305,31 @@ class AutogenerateDiffTestWSchema(AutogenTest, TestCase): connection = self.context.bind diffs = [] autogenerate._produce_net_changes(connection, metadata, diffs, - self.autogen_context) + self.autogen_context, + include_symbol=_default_include_symbol, + include_schemas=True + ) eq_( diffs[0], - ('add_table', metadata.tables['foo.item']) + ('add_table', metadata.tables['%s.item' % self.test_schema_name]) ) eq_(diffs[1][0], 'remove_table') eq_(diffs[1][1].name, "extra") eq_(diffs[2][0], "add_column") - eq_(diffs[2][1], "foo") + eq_(diffs[2][1], self.test_schema_name) eq_(diffs[2][2], "address") - eq_(diffs[2][3], metadata.tables['foo.address'].c.street) + eq_(diffs[2][3], metadata.tables['%s.address' % self.test_schema_name].c.street) eq_(diffs[3][0], "add_column") - eq_(diffs[3][1], "foo") + eq_(diffs[3][1], self.test_schema_name) eq_(diffs[3][2], "order") - eq_(diffs[3][3], metadata.tables['foo.order'].c.user_id) + eq_(diffs[3][3], metadata.tables['%s.order' % self.test_schema_name].c.user_id) eq_(diffs[4][0][0], "modify_type") - eq_(diffs[4][0][1], "foo") + eq_(diffs[4][0][1], self.test_schema_name) eq_(diffs[4][0][2], "order") eq_(diffs[4][0][3], "amount") eq_(repr(diffs[4][0][5]), "NUMERIC(precision=8, scale=2)") @@ -253,7 +340,7 @@ class AutogenerateDiffTestWSchema(AutogenTest, TestCase): eq_(diffs[5][3].name, 'pw') eq_(diffs[6][0][0], "modify_default") - eq_(diffs[6][0][1], "foo") + eq_(diffs[6][0][1], self.test_schema_name) eq_(diffs[6][0][2], "user") eq_(diffs[6][0][3], "a1") eq_(diffs[6][0][6].arg, "x") @@ -264,19 +351,21 @@ class AutogenerateDiffTestWSchema(AutogenTest, TestCase): def test_render_nothing(self): context = MigrationContext.configure( - connection = self.bind.connect(), - opts = { - 'compare_type' : True, - 'compare_server_default' : True, - 'target_metadata' : self.m1, - 'upgrade_token':"upgrades", - 'downgrade_token':"downgrades", + connection=self.bind.connect(), + opts={ + 'compare_type': True, + 'compare_server_default': True, + 'target_metadata': self.m1, + 'upgrade_token': "upgrades", + 'downgrade_token': "downgrades", 'alembic_module_prefix': 'op.', 'sqlalchemy_module_prefix': 'sa.', } ) template_args = {} - autogenerate._produce_migration_diffs(context, template_args, set()) + autogenerate._produce_migration_diffs(context, template_args, set(), + include_symbol=lambda name, schema: False + ) eq_(re.sub(r"u'", "'", template_args['upgrades']), """### commands auto generated by Alembic - please adjust! ### pass @@ -290,7 +379,11 @@ class AutogenerateDiffTestWSchema(AutogenTest, TestCase): """test a full render including indentation""" template_args = {} - autogenerate._produce_migration_diffs(self.context, template_args, set()) + autogenerate._produce_migration_diffs( + self.context, template_args, set(), + include_symbol=_default_include_symbol, + include_schemas=True + ) eq_(re.sub(r"u'", "'", template_args['upgrades']), """### commands auto generated by Alembic - please adjust! ### op.create_table('item', @@ -298,59 +391,59 @@ class AutogenerateDiffTestWSchema(AutogenTest, TestCase): sa.Column('description', sa.String(length=100), nullable=True), sa.Column('order_id', sa.Integer(), nullable=True), sa.CheckConstraint('len(description) > 5'), - sa.ForeignKeyConstraint(['order_id'], ['foo.order.order_id'], ), + sa.ForeignKeyConstraint(['order_id'], ['order.order_id'], ), sa.PrimaryKeyConstraint('id'), - schema='foo' + schema='%(schema)s' ) - op.drop_table('extra', schema='foo') - op.add_column('address', sa.Column('street', sa.String(length=50), nullable=True), schema='foo') - op.add_column('order', sa.Column('user_id', sa.Integer(), nullable=True), schema='foo') + op.drop_table('extra', schema='%(schema)s') + op.add_column('address', sa.Column('street', sa.String(length=50), nullable=True), schema='%(schema)s') + op.add_column('order', sa.Column('user_id', sa.Integer(), nullable=True), schema='%(schema)s') op.alter_column('order', 'amount', existing_type=sa.NUMERIC(precision=8, scale=2), type_=sa.Numeric(precision=10, scale=2), nullable=True, existing_server_default='0::numeric', - schema='foo') - op.drop_column('user', 'pw', schema='foo') + schema='%(schema)s') + op.drop_column('user', 'pw', schema='%(schema)s') op.alter_column('user', 'a1', existing_type=sa.TEXT(), server_default='x', existing_nullable=True, - schema='foo') + schema='%(schema)s') op.alter_column('user', 'name', existing_type=sa.VARCHAR(length=50), nullable=False, - schema='foo') - ### end Alembic commands ###""") + schema='%(schema)s') + ### end Alembic commands ###""" % {"schema": self.test_schema_name}) eq_(re.sub(r"u'", "'", template_args['downgrades']), """### commands auto generated by Alembic - please adjust! ### op.alter_column('user', 'name', existing_type=sa.VARCHAR(length=50), nullable=True, - schema='foo') + schema='%(schema)s') op.alter_column('user', 'a1', existing_type=sa.TEXT(), server_default=None, existing_nullable=True, - schema='foo') - op.add_column('user', sa.Column('pw', sa.VARCHAR(length=50), nullable=True), schema='foo') + schema='%(schema)s') + op.add_column('user', sa.Column('pw', sa.VARCHAR(length=50), nullable=True), schema='%(schema)s') op.alter_column('order', 'amount', existing_type=sa.Numeric(precision=10, scale=2), type_=sa.NUMERIC(precision=8, scale=2), nullable=False, existing_server_default='0::numeric', - schema='foo') - op.drop_column('order', 'user_id', schema='foo') - op.drop_column('address', 'street', schema='foo') + schema='%(schema)s') + op.drop_column('order', 'user_id', schema='%(schema)s') + op.drop_column('address', 'street', schema='%(schema)s') op.create_table('extra', sa.Column('x', sa.CHAR(length=1), nullable=True), sa.Column('uid', sa.INTEGER(), nullable=True), - sa.ForeignKeyConstraint(['uid'], ['foo.user.id'], ), + sa.ForeignKeyConstraint(['uid'], ['%(schema)s.user.id'], name='extra_uid_fkey'), sa.PrimaryKeyConstraint(), - schema='foo' + schema='%(schema)s' ) - op.drop_table('item', schema='foo') - ### end Alembic commands ###""") + op.drop_table('item', schema='%(schema)s') + ### end Alembic commands ###""" % {"schema": self.test_schema_name}) class AutogenerateDiffTest(AutogenTest, TestCase): @@ -369,7 +462,9 @@ class AutogenerateDiffTest(AutogenTest, TestCase): connection = self.context.bind diffs = [] autogenerate._produce_net_changes(connection, metadata, diffs, - self.autogen_context) + self.autogen_context, + include_symbol= _default_include_symbol + ) eq_( diffs[0], @@ -412,13 +507,13 @@ class AutogenerateDiffTest(AutogenTest, TestCase): def test_render_nothing(self): context = MigrationContext.configure( - connection = self.bind.connect(), - opts = { - 'compare_type' : True, - 'compare_server_default' : True, - 'target_metadata' : self.m1, - 'upgrade_token':"upgrades", - 'downgrade_token':"downgrades", + connection=self.bind.connect(), + opts={ + 'compare_type': True, + 'compare_server_default': True, + 'target_metadata': self.m1, + 'upgrade_token': "upgrades", + 'downgrade_token': "downgrades", } ) template_args = {} @@ -590,10 +685,10 @@ class AutogenerateDiffOrderTest(TestCase): connection = empty_context.bind cls.autogen_empty_context = { - 'imports':set(), - 'connection':connection, - 'dialect':connection.dialect, - 'context':empty_context + 'imports': set(), + 'connection': connection, + 'dialect': connection.dialect, + 'context': empty_context } @classmethod @@ -625,11 +720,11 @@ class AutogenRenderTest(TestCase): @requires_07 def setup_class(cls): cls.autogen_context = { - 'opts':{ - 'sqlalchemy_module_prefix' : 'sa.', - 'alembic_module_prefix' : 'op.', + 'opts': { + 'sqlalchemy_module_prefix': 'sa.', + 'alembic_module_prefix': 'op.', }, - 'dialect':mysql.dialect() + 'dialect': mysql.dialect() } def test_render_table_upgrade(self):