From: Bruno Binet Date: Wed, 26 Sep 2012 22:11:01 +0000 (+0200) Subject: add support of schemas for autogenerate X-Git-Tag: rel_0_4_0~4^2^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6a323952147c2c213386f8977bb4e5ba32ff8673;p=thirdparty%2Fsqlalchemy%2Falembic.git add support of schemas for autogenerate for some reasons one of the new tests fails --- diff --git a/alembic/autogenerate.py b/alembic/autogenerate.py index faa2dc74..ebb8396b 100644 --- a/alembic/autogenerate.py +++ b/alembic/autogenerate.py @@ -73,12 +73,15 @@ def compare_metadata(context, metadata): Table(u'bar', MetaData(bind=None), Column(u'data', VARCHAR(), table=), schema=None)), ( 'add_column', + None, 'foo', Column('data', Integer(), table=)), ( 'remove_column', + None, 'foo', Column(u'old_data', VARCHAR(), table=None)), [ ( 'modify_nullable', + None, 'foo', u'x', { 'existing_server_default': None, @@ -150,53 +153,64 @@ def _produce_net_changes(connection, metadata, diffs, autogen_context, include_symbol=None): inspector = Inspector.from_engine(connection) # TODO: not hardcode alembic_version here ? - conn_table_names = set(inspector.get_table_names()).\ - difference(['alembic_version']) - - - metadata_table_names = OrderedSet([table.name + conn_table_names = set() + schemas = inspector.get_schema_names() or [None] + for s in schemas: + if s == 'information_schema': + # ignore postgres own information_schema + continue + tables = set(inspector.get_table_names(schema=s)).\ + difference(['alembic_version']) + conn_table_names.update(zip([s] * len(tables), tables)) + + metadata_table_names = OrderedSet([(table.schema, table.name) for table in metadata.sorted_tables]) if include_symbol: - conn_table_names = set(name for name in conn_table_names - if include_symbol(name)) - metadata_table_names = OrderedSet(name for name in metadata_table_names - if include_symbol(name)) + conn_table_names = set((s, name) + for s, name in conn_table_names + if include_symbol(name, schema=s)) + metadata_table_names = OrderedSet((s, name) + for s, name in metadata_table_names + if include_symbol(name, schema=s)) _compare_tables(conn_table_names, metadata_table_names, inspector, metadata, diffs, autogen_context) def _compare_tables(conn_table_names, metadata_table_names, inspector, metadata, diffs, autogen_context): - for tname in metadata_table_names.difference(conn_table_names): - diffs.append(("add_table", metadata.tables[tname])) - log.info("Detected added table %r", tname) + for s, tname in metadata_table_names.difference(conn_table_names): + name = '%s.%s' % (s, tname) if s else tname + diffs.append(("add_table", metadata.tables[name])) + log.info("Detected added table %r", name) removal_metadata = sa_schema.MetaData() - for tname in conn_table_names.difference(metadata_table_names): - exists = tname in removal_metadata.tables - t = sa_schema.Table(tname, removal_metadata) + for s, tname in conn_table_names.difference(metadata_table_names): + name = '%s.%s' % (s, tname) if s else tname + exists = name in removal_metadata.tables + t = sa_schema.Table(tname, removal_metadata, schema=s) if not exists: inspector.reflecttable(t, None) diffs.append(("remove_table", t)) - log.info("Detected removed table %r", tname) + log.info("Detected removed table %r", name) existing_tables = conn_table_names.intersection(metadata_table_names) conn_column_info = dict( - (tname, + ((s, tname), dict( (rec["name"], rec) - for rec in inspector.get_columns(tname) + for rec in inspector.get_columns(tname, schema=s) ) ) - for tname in existing_tables + for s, tname in existing_tables ) - for tname in sorted(existing_tables): - _compare_columns(tname, - conn_column_info[tname], - metadata.tables[tname], + for s, tname in sorted(existing_tables): + name = '%s.%s' % (s, tname) if s else tname + _compare_columns(s, tname, + conn_column_info[(s, tname)], + metadata.tables[name], diffs, autogen_context) # TODO: @@ -207,44 +221,45 @@ def _compare_tables(conn_table_names, metadata_table_names, ################################################### # element comparison -def _compare_columns(tname, conn_table, metadata_table, +def _compare_columns(schema, tname, conn_table, metadata_table, diffs, autogen_context): + name = '%s.%s' % (schema, tname) if schema else tname metadata_cols_by_name = dict((c.name, c) for c in metadata_table.c) conn_col_names = set(conn_table) metadata_col_names = set(metadata_cols_by_name) for cname in metadata_col_names.difference(conn_col_names): diffs.append( - ("add_column", tname, metadata_cols_by_name[cname]) + ("add_column", schema, tname, metadata_cols_by_name[cname]) ) - log.info("Detected added column '%s.%s'", tname, cname) + log.info("Detected added column '%s.%s'", name, cname) for cname in conn_col_names.difference(metadata_col_names): diffs.append( - ("remove_column", tname, sa_schema.Column( + ("remove_column", schema, tname, sa_schema.Column( cname, conn_table[cname]['type'], nullable=conn_table[cname]['nullable'], server_default=conn_table[cname]['default'] )) ) - log.info("Detected removed column '%s.%s'", tname, cname) + log.info("Detected removed column '%s.%s'", name, cname) for colname in metadata_col_names.intersection(conn_col_names): metadata_col = metadata_table.c[colname] conn_col = conn_table[colname] col_diff = [] - _compare_type(tname, colname, + _compare_type(schema, tname, colname, conn_col, metadata_col, col_diff, autogen_context ) - _compare_nullable(tname, colname, + _compare_nullable(schema, tname, colname, conn_col, metadata_col.nullable, col_diff, autogen_context ) - _compare_server_default(tname, colname, + _compare_server_default(schema, tname, colname, conn_col, metadata_col, col_diff, autogen_context @@ -252,13 +267,13 @@ def _compare_columns(tname, conn_table, metadata_table, if col_diff: diffs.append(col_diff) -def _compare_nullable(tname, cname, conn_col, +def _compare_nullable(schema, tname, cname, conn_col, metadata_col_nullable, diffs, autogen_context): conn_col_nullable = conn_col['nullable'] if conn_col_nullable is not metadata_col_nullable: diffs.append( - ("modify_nullable", tname, cname, + ("modify_nullable", schema, tname, cname, { "existing_type": conn_col['type'], "existing_server_default": conn_col['default'], @@ -272,7 +287,7 @@ def _compare_nullable(tname, cname, conn_col, cname ) -def _compare_type(tname, cname, conn_col, +def _compare_type(schema, tname, cname, conn_col, metadata_col, diffs, autogen_context): @@ -292,7 +307,7 @@ def _compare_type(tname, cname, conn_col, if isdiff: diffs.append( - ("modify_type", tname, cname, + ("modify_type", schema, tname, cname, { "existing_nullable": conn_col['nullable'], "existing_server_default": conn_col['default'], @@ -304,7 +319,7 @@ def _compare_type(tname, cname, conn_col, conn_type, metadata_type, tname, cname ) -def _compare_server_default(tname, cname, conn_col, metadata_col, +def _compare_server_default(schema, tname, cname, conn_col, metadata_col, diffs, autogen_context): metadata_default = metadata_col.server_default @@ -320,7 +335,7 @@ def _compare_server_default(tname, cname, conn_col, metadata_col, if isdiff: conn_col_default = conn_col['default'] diffs.append( - ("modify_default", tname, cname, + ("modify_default", schema, tname, cname, { "existing_nullable": conn_col['nullable'], "existing_type": conn_col['type'], @@ -382,7 +397,7 @@ def _invoke_adddrop_command(updown, args, autogen_context): return cmd_callables[0](*cmd_args) def _invoke_modify_command(updown, args, autogen_context): - tname, cname = args[0][1:3] + sname, tname, cname = args[0][1:4] kw = {} _arg_struct = { @@ -391,7 +406,7 @@ def _invoke_modify_command(updown, args, autogen_context): "modify_default": ("existing_server_default", "server_default"), } for diff in args: - diff_kw = diff[3] + diff_kw = diff[4] for arg in ("existing_type", \ "existing_nullable", \ "existing_server_default"): @@ -409,13 +424,13 @@ def _invoke_modify_command(updown, args, autogen_context): kw.pop("existing_nullable", None) if "server_default" in kw: kw.pop("existing_server_default", None) - return _modify_col(tname, cname, autogen_context, **kw) + return _modify_col(tname, cname, autogen_context, schema=sname, **kw) ################################################### # render python def _add_table(table, autogen_context): - return "%(prefix)screate_table(%(tablename)r,\n%(args)s\n)" % { + text = "%(prefix)screate_table(%(tablename)r,\n%(args)s" % { 'tablename': table.name, 'prefix': _alembic_autogenerate_prefix(autogen_context), 'args': ',\n'.join( @@ -425,28 +440,44 @@ def _add_table(table, autogen_context): table.constraints] if rcons is not None ]) - ), + ) } + if table.schema: + text += ",\nschema=%r" % table.schema + text += "\n)" + return text def _drop_table(table, autogen_context): - return "%(prefix)sdrop_table(%(tname)r)" % { + text = "%(prefix)sdrop_table(%(tname)r" % { "prefix": _alembic_autogenerate_prefix(autogen_context), "tname": table.name } + if table.schema: + text += ", schema=%r" % table.schema + text += ")" + return text -def _add_column(tname, column, autogen_context): - return "%(prefix)sadd_column(%(tname)r, %(column)s)" % { +def _add_column(schema, tname, column, autogen_context): + text = "%(prefix)sadd_column(%(tname)r, %(column)s" % { "prefix": _alembic_autogenerate_prefix(autogen_context), "tname": tname, "column": _render_column(column, autogen_context) } + if schema: + text += ", schema=%r" % schema + text += ")" + return text -def _drop_column(tname, column, autogen_context): - return "%(prefix)sdrop_column(%(tname)r, %(cname)r)" % { +def _drop_column(schema, tname, column, autogen_context): + text = "%(prefix)sdrop_column(%(tname)r, %(cname)r" % { "prefix": _alembic_autogenerate_prefix(autogen_context), "tname": tname, "cname": column.name } + if schema: + text += ", schema=%r" % schema + text += ")" + return text def _modify_col(tname, cname, autogen_context, @@ -455,7 +486,8 @@ def _modify_col(tname, cname, nullable=None, existing_type=None, existing_nullable=None, - existing_server_default=False): + existing_server_default=False, + schema=None): sqla_prefix = _sqlalchemy_autogenerate_prefix(autogen_context) indent = " " * 11 text = "%(prefix)salter_column(%(tname)r, %(cname)r" % { @@ -485,6 +517,8 @@ def _modify_col(tname, cname, existing_server_default, autogen_context), ) + if schema: + text += ",\n%sschema=%r" % (indent, schema) text += ")" return text diff --git a/tests/test_autogenerate.py b/tests/test_autogenerate.py index 7adb4756..49ddf5bb 100644 --- a/tests/test_autogenerate.py +++ b/tests/test_autogenerate.py @@ -15,8 +15,8 @@ import re import sys py3k = sys.version_info >= (3, ) -def _model_one(): - m = MetaData() +def _model_one(schema=None): + m = MetaData(schema=schema) Table('user', m, Column('id', Integer, primary_key=True), @@ -43,8 +43,8 @@ def _model_one(): return m -def _model_two(): - m = MetaData() +def _model_two(schema=None): + m = MetaData(schema=schema) Table('user', m, Column('id', Integer, primary_key=True), @@ -165,7 +165,7 @@ class ImplicitConstraintNoGenTest(AutogenTest, TestCase): template_args = {} autogenerate._produce_migration_diffs(self.context, template_args, set(), - include_symbol=lambda name: name == 'sometable') + include_symbol=lambda name, schema=None: name == 'sometable') eq_( re.sub(r"u'", "'", template_args['upgrades']), "### commands auto generated by Alembic - please adjust! ###\n" @@ -199,6 +199,159 @@ class ImplicitConstraintNoGenTest(AutogenTest, TestCase): +class AutogenerateDiffTestWSchema(AutogenTest, TestCase): + + @classmethod + def _get_bind(cls): + return db_for_dialect('postgresql') + + @classmethod + def _get_db_schema(cls): + return _model_one(schema='foo') + + @classmethod + def _get_model_schema(cls): + return _model_two(schema='foo') + + def test_diffs(self): + """test generation of diff rules""" + + metadata = self.m2 + connection = self.context.bind + diffs = [] + autogenerate._produce_net_changes(connection, metadata, diffs, + self.autogen_context) + + eq_( + diffs[0], + ('add_table', metadata.tables['foo.item']) + ) + + eq_(diffs[1][0], 'remove_table') + eq_(diffs[1][1].name, "extra") + + eq_(diffs[2][0], "add_column") + eq_(diffs[2][1], "foo") + eq_(diffs[2][2], "address") + eq_(diffs[2][3], metadata.tables['foo.address'].c.street) + + eq_(diffs[3][0], "add_column") + eq_(diffs[3][1], "foo") + eq_(diffs[3][2], "order") + eq_(diffs[3][3], metadata.tables['foo.order'].c.user_id) + + eq_(diffs[4][0][0], "modify_type") + eq_(diffs[4][0][1], "foo") + eq_(diffs[4][0][2], "order") + eq_(diffs[4][0][3], "amount") + eq_(repr(diffs[4][0][5]), "NUMERIC(precision=8, scale=2)") + eq_(repr(diffs[4][0][6]), "Numeric(precision=10, scale=2)") + + + eq_(diffs[5][0], 'remove_column') + eq_(diffs[5][3].name, 'pw') + + eq_(diffs[6][0][0], "modify_default") + eq_(diffs[6][0][1], "foo") + eq_(diffs[6][0][2], "user") + eq_(diffs[6][0][3], "a1") + eq_(diffs[6][0][6].arg, "x") + + eq_(diffs[7][0][0], 'modify_nullable') + eq_(diffs[7][0][5], True) + eq_(diffs[7][0][6], False) + + def test_render_nothing(self): + context = MigrationContext.configure( + connection = self.bind.connect(), + opts = { + 'compare_type' : True, + 'compare_server_default' : True, + 'target_metadata' : self.m1, + 'upgrade_token':"upgrades", + 'downgrade_token':"downgrades", + 'alembic_module_prefix': 'op.', + 'sqlalchemy_module_prefix': 'sa.', + } + ) + template_args = {} + autogenerate._produce_migration_diffs(context, template_args, set()) + eq_(re.sub(r"u'", "'", template_args['upgrades']), +"""### commands auto generated by Alembic - please adjust! ### + pass + ### end Alembic commands ###""") + eq_(re.sub(r"u'", "'", template_args['downgrades']), +"""### commands auto generated by Alembic - please adjust! ### + pass + ### end Alembic commands ###""") + + def test_render_diffs(self): + """test a full render including indentation""" + + template_args = {} + autogenerate._produce_migration_diffs(self.context, template_args, set()) + eq_(re.sub(r"u'", "'", template_args['upgrades']), +"""### commands auto generated by Alembic - please adjust! ### + op.create_table('item', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('description', sa.String(length=100), nullable=True), + sa.Column('order_id', sa.Integer(), nullable=True), + sa.CheckConstraint('len(description) > 5'), + sa.ForeignKeyConstraint(['order_id'], ['foo.order.order_id'], ), + sa.PrimaryKeyConstraint('id'), + schema='foo' + ) + op.drop_table('extra', schema='foo') + op.add_column('address', sa.Column('street', sa.String(length=50), nullable=True), schema='foo') + op.add_column('order', sa.Column('user_id', sa.Integer(), nullable=True), schema='foo') + op.alter_column('order', 'amount', + existing_type=sa.NUMERIC(precision=8, scale=2), + type_=sa.Numeric(precision=10, scale=2), + nullable=True, + existing_server_default='0::numeric', + schema='foo') + op.drop_column('user', 'pw', schema='foo') + op.alter_column('user', 'a1', + existing_type=sa.TEXT(), + server_default='x', + existing_nullable=True, + schema='foo') + op.alter_column('user', 'name', + existing_type=sa.VARCHAR(length=50), + nullable=False, + schema='foo') + ### end Alembic commands ###""") + eq_(re.sub(r"u'", "'", template_args['downgrades']), +"""### commands auto generated by Alembic - please adjust! ### + op.alter_column('user', 'name', + existing_type=sa.VARCHAR(length=50), + nullable=True, + schema='foo') + op.alter_column('user', 'a1', + existing_type=sa.TEXT(), + server_default=None, + existing_nullable=True, + schema='foo') + op.add_column('user', sa.Column('pw', sa.VARCHAR(length=50), nullable=True), schema='foo') + op.alter_column('order', 'amount', + existing_type=sa.Numeric(precision=10, scale=2), + type_=sa.NUMERIC(precision=8, scale=2), + nullable=False, + existing_server_default='0::numeric', + schema='foo') + op.drop_column('order', 'user_id', schema='foo') + op.drop_column('address', 'street', schema='foo') + op.create_table('extra', + sa.Column('x', sa.CHAR(length=1), nullable=True), + sa.Column('uid', sa.INTEGER(), nullable=True), + sa.ForeignKeyConstraint(['uid'], ['foo.user.id'], ), + sa.PrimaryKeyConstraint(), + schema='foo' + ) + op.drop_table('item', schema='foo') + ### end Alembic commands ###""") + + class AutogenerateDiffTest(AutogenTest, TestCase): @classmethod def _get_db_schema(cls): @@ -226,31 +379,35 @@ class AutogenerateDiffTest(AutogenTest, TestCase): eq_(diffs[1][1].name, "extra") eq_(diffs[2][0], "add_column") - eq_(diffs[2][1], "address") - eq_(diffs[2][2], metadata.tables['address'].c.street) + eq_(diffs[2][1], None) + eq_(diffs[2][2], "address") + eq_(diffs[2][3], metadata.tables['address'].c.street) eq_(diffs[3][0], "add_column") - eq_(diffs[3][1], "order") - eq_(diffs[3][2], metadata.tables['order'].c.user_id) + eq_(diffs[3][1], None) + eq_(diffs[3][2], "order") + eq_(diffs[3][3], metadata.tables['order'].c.user_id) eq_(diffs[4][0][0], "modify_type") - eq_(diffs[4][0][1], "order") - eq_(diffs[4][0][2], "amount") - eq_(repr(diffs[4][0][4]), "NUMERIC(precision=8, scale=2)") - eq_(repr(diffs[4][0][5]), "Numeric(precision=10, scale=2)") + eq_(diffs[4][0][1], None) + eq_(diffs[4][0][2], "order") + eq_(diffs[4][0][3], "amount") + eq_(repr(diffs[4][0][5]), "NUMERIC(precision=8, scale=2)") + eq_(repr(diffs[4][0][6]), "Numeric(precision=10, scale=2)") eq_(diffs[5][0], 'remove_column') - eq_(diffs[5][2].name, 'pw') + eq_(diffs[5][3].name, 'pw') eq_(diffs[6][0][0], "modify_default") - eq_(diffs[6][0][1], "user") - eq_(diffs[6][0][2], "a1") - eq_(diffs[6][0][5].arg, "x") + eq_(diffs[6][0][1], None) + eq_(diffs[6][0][2], "user") + eq_(diffs[6][0][3], "a1") + eq_(diffs[6][0][6].arg, "x") eq_(diffs[7][0][0], 'modify_nullable') - eq_(diffs[7][0][4], True) - eq_(diffs[7][0][5], False) + eq_(diffs[7][0][5], True) + eq_(diffs[7][0][6], False) def test_render_nothing(self): context = MigrationContext.configure( @@ -357,7 +514,7 @@ class AutogenerateDiffTest(AutogenTest, TestCase): def test_skip_null_type_comparison_reflected(self): diff = [] - autogenerate._compare_type("sometable", "somecol", + autogenerate._compare_type(None, "sometable", "somecol", {"name":"somecol", "type":NULLTYPE, "nullable":True, "default":None}, Column("somecol", Integer()), @@ -367,7 +524,7 @@ class AutogenerateDiffTest(AutogenTest, TestCase): def test_skip_null_type_comparison_local(self): diff = [] - autogenerate._compare_type("sometable", "somecol", + autogenerate._compare_type(None, "sometable", "somecol", {"name":"somecol", "type":Integer(), "nullable":True, "default":None}, Column("somecol", NULLTYPE), @@ -386,7 +543,7 @@ class AutogenerateDiffTest(AutogenTest, TestCase): return dialect.type_descriptor(CHAR(32)) diff = [] - autogenerate._compare_type("sometable", "somecol", + autogenerate._compare_type(None, "sometable", "somecol", {"name":"somecol", "type":Integer(), "nullable":True, "default":None}, Column("somecol", MyType()), @@ -399,7 +556,7 @@ class AutogenerateDiffTest(AutogenTest, TestCase): from sqlalchemy.util import OrderedSet inspector = Inspector.from_engine(self.bind) autogenerate._compare_tables( - OrderedSet(['extra', 'user']), OrderedSet(), inspector, + OrderedSet([(None, 'extra'), (None, 'user')]), OrderedSet(), inspector, MetaData(), diffs, self.autogen_context ) eq_( @@ -502,6 +659,21 @@ class AutogenRenderTest(TestCase): ")" ) + def test_render_table_w_schema(self): + m = MetaData() + t = Table('test', m, + Column('id', Integer, primary_key=True), + schema='foo' + ) + eq_ignore_whitespace( + autogenerate._add_table(t, self.autogen_context), + "op.create_table('test'," + "sa.Column('id', sa.Integer(), nullable=False)," + "sa.PrimaryKeyConstraint('id')," + "schema='foo'" + ")" + ) + def test_render_drop_table(self): eq_( autogenerate._drop_table(Table("sometable", MetaData()), @@ -509,24 +681,50 @@ class AutogenRenderTest(TestCase): "op.drop_table('sometable')" ) + def test_render_drop_table_w_schema(self): + eq_( + autogenerate._drop_table( + Table("sometable", MetaData(), schema='foo'), + self.autogen_context), + "op.drop_table('sometable', schema='foo')" + ) + def test_render_add_column(self): eq_( autogenerate._add_column( - "foo", Column("x", Integer, server_default="5"), + None, "foo", Column("x", Integer, server_default="5"), self.autogen_context), "op.add_column('foo', sa.Column('x', sa.Integer(), " "server_default='5', nullable=True))" ) + def test_render_add_column_w_schema(self): + eq_( + autogenerate._add_column( + "foo", "bar", Column("x", Integer, server_default="5"), + self.autogen_context), + "op.add_column('bar', sa.Column('x', sa.Integer(), " + "server_default='5', nullable=True), schema='foo')" + ) + def test_render_drop_column(self): eq_( autogenerate._drop_column( - "foo", Column("x", Integer, server_default="5"), + None, "foo", Column("x", Integer, server_default="5"), self.autogen_context), "op.drop_column('foo', 'x')" ) + def test_render_drop_column_w_schema(self): + eq_( + autogenerate._drop_column( + "foo", "bar", Column("x", Integer, server_default="5"), + self.autogen_context), + + "op.drop_column('bar', 'x', schema='foo')" + ) + def test_render_quoted_server_default(self): eq_( autogenerate._render_server_default( @@ -559,6 +757,18 @@ class AutogenRenderTest(TestCase): "existing_type=sa.CHAR(length=20), type_=sa.CHAR(length=10))" ) + def test_render_modify_type_w_schema(self): + eq_ignore_whitespace( + autogenerate._modify_col( + "sometable", "somecolumn", + self.autogen_context, + type_=CHAR(10), existing_type=CHAR(20), + schema='foo'), + "op.alter_column('sometable', 'somecolumn', " + "existing_type=sa.CHAR(length=20), type_=sa.CHAR(length=10), " + "schema='foo')" + ) + def test_render_modify_nullable(self): eq_ignore_whitespace( autogenerate._modify_col( @@ -570,6 +780,17 @@ class AutogenRenderTest(TestCase): "existing_type=sa.Integer(), nullable=True)" ) + def test_render_modify_nullable_w_schema(self): + eq_ignore_whitespace( + autogenerate._modify_col( + "sometable", "somecolumn", + self.autogen_context, + existing_type=Integer(), + nullable=True, schema='foo'), + "op.alter_column('sometable', 'somecolumn', " + "existing_type=sa.Integer(), nullable=True, schema='foo')" + ) + def test_render_check_constraint_literal(self): eq_ignore_whitespace( autogenerate._render_check_constraint(