Allow metadata.reflect() to recover from unreflectable tables

Added support for views that are unreflectable due to stale
table definitions, when calling :meth:`.MetaData.reflect`; a warning
is emitted for the table that cannot respond to ``DESCRIBE``
but the operation succeeds.  The MySQL dialect now
raises UnreflectableTableError which is in turn caught by
MetaData.reflect().  Reflecting the view standalone raises
this error directly.

Change-Id: Id8005219d8e073c154cc84a873df911b4a6cf4d6
Fixes: #3871

doc/build/changelog/changelog_12.rst

@@ -25,6 +25,15 @@
 
             :ref:`change_3796`
 
+    .. change:: 3871
+        :tags: bug, mysql
+        :tickets: 3871
+
+        Added support for views that are unreflectable due to stale
+        table definitions, when calling :meth:`.MetaData.reflect`; a warning
+        is emitted for the table that cannot respond to ``DESCRIBE``,
+        but the operation succeeds.
+
     .. change:: baked_opts
         :tags: feature, ext
 

lib/sqlalchemy/dialects/mysql/base.py

@@ -2052,8 +2052,14 @@ class MySQLDialect(default.DefaultDialect):
                 rp = connection.execution_options(
                     skip_user_error_events=True).execute(st)
             except exc.DBAPIError as e:
-                if self._extract_error_code(e.orig) == 1146:
+                code = self._extract_error_code(e.orig)
+                if code == 1146:
                     raise exc.NoSuchTableError(full_name)
+                elif code == 1356:
+                    raise exc.UnreflectableTableError(
+                        "Table or view named %s could not be "
+                        "reflected: %s" % (full_name, e)
+                    )
                 else:
                     raise
             rows = self._compat_fetchall(rp, charset=charset)
@@ -2063,7 +2069,6 @@ class MySQLDialect(default.DefaultDialect):
         return rows
 
 
-
 class _DecodingRowProxy(object):
     """Return unicode-decoded values based on type inspection.
 

lib/sqlalchemy/exc.py

@@ -199,6 +199,14 @@ class NoSuchTableError(InvalidRequestError):
     """Table does not exist or is not visible to a connection."""
 
 
+class UnreflectableTableError(InvalidRequestError):
+    """Table exists but can't be reflectted for some reason.
+
+    .. versionadded:: 1.2
+
+    """
+
+
 class UnboundExecutionError(InvalidRequestError):
     """SQL was attempted without a database connection to execute it on."""
 

lib/sqlalchemy/sql/schema.py

@@ -3903,7 +3903,10 @@ class MetaData(SchemaItem):
                         name not in current]
 
             for name in load:
-                Table(name, self, **reflect_opts)
+                try:
+                    Table(name, self, **reflect_opts)
+                except exc.UnreflectableTableError as uerr:
+                    util.warn("Skipping table %s: %s" % (name, uerr))
 
     def append_ddl_listener(self, event_name, listener):
         """Append a DDL event listener to this ``MetaData``.

lib/sqlalchemy/testing/assertions.py

@@ -130,9 +130,16 @@ def _expect_warnings(exc_cls, messages, regex=True, assert_=True,
 
     real_warn = warnings.warn
 
-    def our_warn(msg, exception, *arg, **kw):
-        if not issubclass(exception, exc_cls):
-            return real_warn(msg, exception, *arg, **kw)
+    def our_warn(msg, *arg, **kw):
+        if isinstance(msg, exc_cls):
+            exception = msg
+            msg = str(exception)
+        elif arg:
+            exception = arg[0]
+        else:
+            exception = None
+        if not exception or not issubclass(exception, exc_cls):
+            return real_warn(msg, *arg, **kw)
 
         if not filters:
             return
@@ -143,7 +150,7 @@ def _expect_warnings(exc_cls, messages, regex=True, assert_=True,
                 seen.discard(filter_)
                 break
         else:
-            real_warn(msg, exception, *arg, **kw)
+            real_warn(msg, *arg, **kw)
 
     with mock.patch("warnings.warn", our_warn):
         yield

test/dialect/mysql/test_reflection.py

@@ -7,11 +7,13 @@ from sqlalchemy import Column, Table, DDL, MetaData, TIMESTAMP, \
     BigInteger
 from sqlalchemy import event
 from sqlalchemy import sql
+from sqlalchemy import exc
 from sqlalchemy import inspect
 from sqlalchemy.dialects.mysql import base as mysql
 from sqlalchemy.dialects.mysql import reflection as _reflection
 from sqlalchemy.testing import fixtures, AssertsExecutionResults
 from sqlalchemy import testing
+from sqlalchemy.testing import assert_raises_message, expect_warnings
 
 
 class TypeReflectionTest(fixtures.TestBase):
@@ -430,6 +432,35 @@ class ReflectionTest(fixtures.TestBase, AssertsExecutionResults):
                 [('a', mysql.INTEGER), ('b', mysql.VARCHAR)]
             )
 
+    @testing.provide_metadata
+    def test_skip_not_describable(self):
+        @event.listens_for(self.metadata, "before_drop")
+        def cleanup(*arg, **kw):
+            with testing.db.connect() as conn:
+                conn.execute("DROP TABLE IF EXISTS test_t1")
+                conn.execute("DROP TABLE IF EXISTS test_t2")
+                conn.execute("DROP VIEW IF EXISTS test_v")
+
+        with testing.db.connect() as conn:
+            conn.execute("CREATE TABLE test_t1 (id INTEGER)")
+            conn.execute("CREATE TABLE test_t2 (id INTEGER)")
+            conn.execute("CREATE VIEW test_v AS SELECT id FROM test_t1" )
+            conn.execute("DROP TABLE test_t1")
+
+            m = MetaData()
+            with expect_warnings(
+                "Skipping .* Table or view named .?test_v.? could not be "
+                "reflected: .* references invalid table"
+            ):
+                m.reflect(views=True, bind=conn)
+            eq_(m.tables['test_t2'].name, "test_t2")
+
+            assert_raises_message(
+                exc.UnreflectableTableError,
+                "references invalid table",
+                Table, 'test_v', MetaData(), autoload_with=conn
+            )
+
     @testing.exclude('mysql', '<', (5, 0, 0), 'no information_schema support')
     def test_system_views(self):
         dialect = testing.db.dialect

test/engine/test_reflection.py

@@ -12,6 +12,8 @@ from sqlalchemy.testing import eq_, is_true, assert_raises, \
 from sqlalchemy import testing
 from sqlalchemy.util import ue
 from sqlalchemy.testing import config
+from sqlalchemy.testing import mock
+from sqlalchemy.testing import expect_warnings
 
 metadata, users = None, None
 
@@ -972,6 +974,35 @@ class ReflectionTest(fixtures.TestBase, ComparesTables):
             m9.reflect()
             self.assert_(not m9.tables)
 
+    @testing.provide_metadata
+    def test_reflect_all_unreflectable_table(self):
+        names = ['rt_%s' % name for name in ('a', 'b', 'c', 'd', 'e')]
+
+        for name in names:
+            Table(name, self.metadata,
+                  Column('id', sa.Integer, primary_key=True))
+        self.metadata.create_all()
+
+        m = MetaData()
+
+        reflecttable = testing.db.dialect.reflecttable
+
+        def patched(conn, table, *arg, **kw):
+            if table.name == 'rt_c':
+                raise sa.exc.UnreflectableTableError("Can't reflect rt_c")
+            else:
+                return reflecttable(conn, table, *arg, **kw)
+
+        with mock.patch.object(testing.db.dialect, "reflecttable", patched):
+            with expect_warnings("Skipping table rt_c: Can't reflect rt_c"):
+                m.reflect(bind=testing.db)
+
+            assert_raises_message(
+                sa.exc.UnreflectableTableError,
+                "Can't reflect rt_c",
+                Table, 'rt_c', m, autoload_with=testing.db
+            )
+
     def test_reflect_all_conn_closing(self):
         m1 = MetaData()
         c = testing.db.connect()