From: Mike Bayer Date: Mon, 22 May 2017 18:08:55 +0000 (-0400) Subject: Allow metadata.reflect() to recover from unreflectable tables X-Git-Tag: rel_1_2_0b1~55 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9f0fb6c601829cb7c9f449d57e12e8b95dab51f5;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Allow metadata.reflect() to recover from unreflectable tables Added support for views that are unreflectable due to stale table definitions, when calling :meth:`.MetaData.reflect`; a warning is emitted for the table that cannot respond to ``DESCRIBE`` but the operation succeeds. The MySQL dialect now raises UnreflectableTableError which is in turn caught by MetaData.reflect(). Reflecting the view standalone raises this error directly. Change-Id: Id8005219d8e073c154cc84a873df911b4a6cf4d6 Fixes: #3871 --- diff --git a/doc/build/changelog/changelog_12.rst b/doc/build/changelog/changelog_12.rst index 0f7b7b9af3..a96436aaf8 100644 --- a/doc/build/changelog/changelog_12.rst +++ b/doc/build/changelog/changelog_12.rst @@ -25,6 +25,15 @@ :ref:`change_3796` + .. change:: 3871 + :tags: bug, mysql + :tickets: 3871 + + Added support for views that are unreflectable due to stale + table definitions, when calling :meth:`.MetaData.reflect`; a warning + is emitted for the table that cannot respond to ``DESCRIBE``, + but the operation succeeds. + .. change:: baked_opts :tags: feature, ext diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 21d325d0ca..277ae58150 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -2052,8 +2052,14 @@ class MySQLDialect(default.DefaultDialect): rp = connection.execution_options( skip_user_error_events=True).execute(st) except exc.DBAPIError as e: - if self._extract_error_code(e.orig) == 1146: + code = self._extract_error_code(e.orig) + if code == 1146: raise exc.NoSuchTableError(full_name) + elif code == 1356: + raise exc.UnreflectableTableError( + "Table or view named %s could not be " + "reflected: %s" % (full_name, e) + ) else: raise rows = self._compat_fetchall(rp, charset=charset) @@ -2063,7 +2069,6 @@ class MySQLDialect(default.DefaultDialect): return rows - class _DecodingRowProxy(object): """Return unicode-decoded values based on type inspection. diff --git a/lib/sqlalchemy/exc.py b/lib/sqlalchemy/exc.py index e8ba34ba4e..3db3c1085d 100644 --- a/lib/sqlalchemy/exc.py +++ b/lib/sqlalchemy/exc.py @@ -199,6 +199,14 @@ class NoSuchTableError(InvalidRequestError): """Table does not exist or is not visible to a connection.""" +class UnreflectableTableError(InvalidRequestError): + """Table exists but can't be reflectted for some reason. + + .. versionadded:: 1.2 + + """ + + class UnboundExecutionError(InvalidRequestError): """SQL was attempted without a database connection to execute it on.""" diff --git a/lib/sqlalchemy/sql/schema.py b/lib/sqlalchemy/sql/schema.py index 1cdc7b4250..a9aee58835 100644 --- a/lib/sqlalchemy/sql/schema.py +++ b/lib/sqlalchemy/sql/schema.py @@ -3903,7 +3903,10 @@ class MetaData(SchemaItem): name not in current] for name in load: - Table(name, self, **reflect_opts) + try: + Table(name, self, **reflect_opts) + except exc.UnreflectableTableError as uerr: + util.warn("Skipping table %s: %s" % (name, uerr)) def append_ddl_listener(self, event_name, listener): """Append a DDL event listener to this ``MetaData``. diff --git a/lib/sqlalchemy/testing/assertions.py b/lib/sqlalchemy/testing/assertions.py index 8845563457..dfea33dc79 100644 --- a/lib/sqlalchemy/testing/assertions.py +++ b/lib/sqlalchemy/testing/assertions.py @@ -130,9 +130,16 @@ def _expect_warnings(exc_cls, messages, regex=True, assert_=True, real_warn = warnings.warn - def our_warn(msg, exception, *arg, **kw): - if not issubclass(exception, exc_cls): - return real_warn(msg, exception, *arg, **kw) + def our_warn(msg, *arg, **kw): + if isinstance(msg, exc_cls): + exception = msg + msg = str(exception) + elif arg: + exception = arg[0] + else: + exception = None + if not exception or not issubclass(exception, exc_cls): + return real_warn(msg, *arg, **kw) if not filters: return @@ -143,7 +150,7 @@ def _expect_warnings(exc_cls, messages, regex=True, assert_=True, seen.discard(filter_) break else: - real_warn(msg, exception, *arg, **kw) + real_warn(msg, *arg, **kw) with mock.patch("warnings.warn", our_warn): yield diff --git a/test/dialect/mysql/test_reflection.py b/test/dialect/mysql/test_reflection.py index 9e122e680f..dc088223d8 100644 --- a/test/dialect/mysql/test_reflection.py +++ b/test/dialect/mysql/test_reflection.py @@ -7,11 +7,13 @@ from sqlalchemy import Column, Table, DDL, MetaData, TIMESTAMP, \ BigInteger from sqlalchemy import event from sqlalchemy import sql +from sqlalchemy import exc from sqlalchemy import inspect from sqlalchemy.dialects.mysql import base as mysql from sqlalchemy.dialects.mysql import reflection as _reflection from sqlalchemy.testing import fixtures, AssertsExecutionResults from sqlalchemy import testing +from sqlalchemy.testing import assert_raises_message, expect_warnings class TypeReflectionTest(fixtures.TestBase): @@ -430,6 +432,35 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults): [('a', mysql.INTEGER), ('b', mysql.VARCHAR)] ) + @testing.provide_metadata + def test_skip_not_describable(self): + @event.listens_for(self.metadata, "before_drop") + def cleanup(*arg, **kw): + with testing.db.connect() as conn: + conn.execute("DROP TABLE IF EXISTS test_t1") + conn.execute("DROP TABLE IF EXISTS test_t2") + conn.execute("DROP VIEW IF EXISTS test_v") + + with testing.db.connect() as conn: + conn.execute("CREATE TABLE test_t1 (id INTEGER)") + conn.execute("CREATE TABLE test_t2 (id INTEGER)") + conn.execute("CREATE VIEW test_v AS SELECT id FROM test_t1" ) + conn.execute("DROP TABLE test_t1") + + m = MetaData() + with expect_warnings( + "Skipping .* Table or view named .?test_v.? could not be " + "reflected: .* references invalid table" + ): + m.reflect(views=True, bind=conn) + eq_(m.tables['test_t2'].name, "test_t2") + + assert_raises_message( + exc.UnreflectableTableError, + "references invalid table", + Table, 'test_v', MetaData(), autoload_with=conn + ) + @testing.exclude('mysql', '<', (5, 0, 0), 'no information_schema support') def test_system_views(self): dialect = testing.db.dialect diff --git a/test/engine/test_reflection.py b/test/engine/test_reflection.py index 9616c300da..80a8047965 100644 --- a/test/engine/test_reflection.py +++ b/test/engine/test_reflection.py @@ -12,6 +12,8 @@ from sqlalchemy.testing import eq_, is_true, assert_raises, \ from sqlalchemy import testing from sqlalchemy.util import ue from sqlalchemy.testing import config +from sqlalchemy.testing import mock +from sqlalchemy.testing import expect_warnings metadata, users = None, None @@ -972,6 +974,35 @@ class ReflectionTest(fixtures.TestBase, ComparesTables): m9.reflect() self.assert_(not m9.tables) + @testing.provide_metadata + def test_reflect_all_unreflectable_table(self): + names = ['rt_%s' % name for name in ('a', 'b', 'c', 'd', 'e')] + + for name in names: + Table(name, self.metadata, + Column('id', sa.Integer, primary_key=True)) + self.metadata.create_all() + + m = MetaData() + + reflecttable = testing.db.dialect.reflecttable + + def patched(conn, table, *arg, **kw): + if table.name == 'rt_c': + raise sa.exc.UnreflectableTableError("Can't reflect rt_c") + else: + return reflecttable(conn, table, *arg, **kw) + + with mock.patch.object(testing.db.dialect, "reflecttable", patched): + with expect_warnings("Skipping table rt_c: Can't reflect rt_c"): + m.reflect(bind=testing.db) + + assert_raises_message( + sa.exc.UnreflectableTableError, + "Can't reflect rt_c", + Table, 'rt_c', m, autoload_with=testing.db + ) + def test_reflect_all_conn_closing(self): m1 = MetaData() c = testing.db.connect()