Fix tests failing for SQLite file databases; repair provisioning

1. ensure provision.py loads dialect implementations when running
reap_dbs.py.   Reapers haven't been working since
598f2f7e557073f29563d4d567f43931fc03013f .

2. add some exclusion rules to allow the sqlite_file target to work;
add to tox.

3. add reap dbs target for SQLite, repair SQLite drop_db routine
which also wasn't doing the right thing for memory databases
etc.

4. Fix logging in provision files, as the main provision logger
is the one that's enabled by reap_dbs and maybe others, have all
the provision files use the provision logger.

Fixes: #5180
Fixes: #5168
Change-Id: Ibc1b0106394d20f5bcf847f37b09d185f26ac9b5

diff --git a/lib/sqlalchemy/dialects/mssql/provision.py b/lib/sqlalchemy/dialects/mssql/provision.py
index 558ad7a7ab823b43cea6e1529fd1beb64649b5ca..a10043db15578a629232af907f01f320fd0afd9b 100644 (file)
--- a/lib/sqlalchemy/dialects/mssql/provision.py
+++ b/lib/sqlalchemy/dialects/mssql/provision.py
@@ -1,16 +1,12 @@
-import logging
-
 from ... import create_engine
 from ... import exc
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
+from ...testing.provision import log
 from ...testing.provision import run_reap_dbs
 from ...testing.provision import update_db_opts
 
 
-log = logging.getLogger(__name__)
-
-
 @update_db_opts.for_db("mssql")
 def _mssql_update_db_opts(db_url, db_opts):
     db_opts["legacy_schema_aliasing"] = False

diff --git a/lib/sqlalchemy/dialects/oracle/provision.py b/lib/sqlalchemy/dialects/oracle/provision.py
index 8fc2ac23524ec3d88d726c7cf718bd45b5b85f58..7901eb4e813bc62439b385f6f317294cffa84a67 100644 (file)
--- a/lib/sqlalchemy/dialects/oracle/provision.py
+++ b/lib/sqlalchemy/dialects/oracle/provision.py
@@ -1,5 +1,3 @@
-import logging
-
 from ... import create_engine
 from ... import exc
 from ...engine import url as sa_url
@@ -7,14 +5,12 @@ from ...testing.provision import configure_follower
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
 from ...testing.provision import follower_url_from_main
+from ...testing.provision import log
 from ...testing.provision import run_reap_dbs
 from ...testing.provision import temp_table_keyword_args
 from ...testing.provision import update_db_opts
 
 
-log = logging.getLogger(__name__)
-
-
 @create_db.for_db("oracle")
 def _oracle_create_db(cfg, eng, ident):
     # NOTE: make sure you've run "ALTER DATABASE default tablespace users" or

diff --git a/lib/sqlalchemy/dialects/postgresql/provision.py b/lib/sqlalchemy/dialects/postgresql/provision.py
index 404da93a8fa5fca6e72bad73c0d3c3339572d1a4..c8f83d2bef0a54bdaa6648ba68bca5ed25db932a 100644 (file)
--- a/lib/sqlalchemy/dialects/postgresql/provision.py
+++ b/lib/sqlalchemy/dialects/postgresql/provision.py
@@ -1,16 +1,13 @@
-import logging
 import time
 
 from ... import exc
 from ... import text
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
+from ...testing.provision import log
 from ...testing.provision import temp_table_keyword_args
 
 
-log = logging.getLogger(__name__)
-
-
 @create_db.for_db("postgresql")
 def _pg_create_db(cfg, eng, ident):
     template_db = cfg.options.postgresql_templatedb

diff --git a/lib/sqlalchemy/dialects/sqlite/provision.py b/lib/sqlalchemy/dialects/sqlite/provision.py
index d4a5ae93f50c05f145ab88d5546bb08292ee9351..ce20ed991234eb7a3e96ed3583c35271b9027c50 100644 (file)
--- a/lib/sqlalchemy/dialects/sqlite/provision.py
+++ b/lib/sqlalchemy/dialects/sqlite/provision.py
@@ -4,7 +4,9 @@ from ...engine import url as sa_url
 from ...testing.provision import create_db
 from ...testing.provision import drop_db
 from ...testing.provision import follower_url_from_main
+from ...testing.provision import log
 from ...testing.provision import post_configure_engine
+from ...testing.provision import run_reap_dbs
 from ...testing.provision import temp_table_keyword_args
 
 
@@ -26,6 +28,11 @@ def _sqlite_post_configure_engine(url, engine, follower_ident):
         # use file DBs in all cases, memory acts kind of strangely
         # as an attached
         if not follower_ident:
+            # note this test_schema.db gets created for all test runs.
+            # there's not any dedicated cleanup step for it.  it in some
+            # ways corresponds to the "test.test_schema" schema that's
+            # expected to be already present, so for now it just stays
+            # in a given checkout directory.
             dbapi_connection.execute(
                 'ATTACH DATABASE "test_schema.db" AS test_schema'
             )
@@ -43,12 +50,26 @@ def _sqlite_create_db(cfg, eng, ident):
 
 @drop_db.for_db("sqlite")
 def _sqlite_drop_db(cfg, eng, ident):
-    if ident:
-        os.remove("%s_test_schema.db" % ident)
-    else:
-        os.remove("%s.db" % ident)
+    for path in ["%s.db" % ident, "%s_test_schema.db" % ident]:
+        if os.path.exists(path):
+            log.info("deleting SQLite database file: %s" % path)
+            os.remove(path)
 
 
 @temp_table_keyword_args.for_db("sqlite")
 def _sqlite_temp_table_keyword_args(cfg, eng):
     return {"prefixes": ["TEMPORARY"]}
+
+
+@run_reap_dbs.for_db("sqlite")
+def _reap_sqlite_dbs(url, idents):
+    log.info("db reaper connecting to %r", url)
+
+    log.info("identifiers in file: %s", ", ".join(idents))
+    for ident in idents:
+        # we don't have a config so we can't call _sqlite_drop_db due to the
+        # decorator
+        for path in ["%s.db" % ident, "%s_test_schema.db" % ident]:
+            if os.path.exists(path):
+                log.info("deleting SQLite database file: %s" % path)
+                os.remove(path)

diff --git a/lib/sqlalchemy/dialects/sqlite/pysqlite.py b/lib/sqlalchemy/dialects/sqlite/pysqlite.py
index 0585e69ad4764e748cb82d0177675b8de26bc13c..4485631ce56f035be4e7d78009275ccb7dea3f13 100644 (file)
--- a/lib/sqlalchemy/dialects/sqlite/pysqlite.py
+++ b/lib/sqlalchemy/dialects/sqlite/pysqlite.py
@@ -421,8 +421,15 @@ class SQLiteDialect_pysqlite(SQLiteDialect):
         return sqlite
 
     @classmethod
-    def get_pool_class(cls, url):
+    def _is_url_file_db(cls, url):
         if url.database and url.database != ":memory:":
+            return True
+        else:
+            return False
+
+    @classmethod
+    def get_pool_class(cls, url):
+        if cls._is_url_file_db(url):
             return pool.NullPool
         else:
             return pool.SingletonThreadPool

diff --git a/lib/sqlalchemy/testing/profiling.py b/lib/sqlalchemy/testing/profiling.py
index 05a0fde49edc927522ff791c1667cae405ae905d..b6108400d25b7c545d923c6bd5eda9dd60e1b64d 100644 (file)
--- a/lib/sqlalchemy/testing/profiling.py
+++ b/lib/sqlalchemy/testing/profiling.py
@@ -88,6 +88,11 @@ class ProfileStatsFile(object):
 
         dbapi_key = config.db.name + "_" + config.db.driver
 
+        if config.db.name == "sqlite" and config.db.dialect._is_url_file_db(
+            config.db.url
+        ):
+            dbapi_key += "_file"
+
         # keep it at 2.7, 3.1, 3.2, etc. for now.
         py_version = ".".join([str(v) for v in sys.version_info[0:2]])
 

diff --git a/lib/sqlalchemy/testing/provision.py b/lib/sqlalchemy/testing/provision.py
index 543d91a533f2e247d944afed870ea9a99741fc2c..0123ea7e7aa93972496293beba314f0ca22833d9 100644 (file)
--- a/lib/sqlalchemy/testing/provision.py
+++ b/lib/sqlalchemy/testing/provision.py
@@ -161,12 +161,16 @@ def reap_dbs(idents_file):
 
     urls = collections.defaultdict(set)
     idents = collections.defaultdict(set)
+    dialects = {}
 
     with open(idents_file) as file_:
         for line in file_:
             line = line.strip()
             db_name, db_url = line.split(" ")
             url_obj = sa_url.make_url(db_url)
+            if db_name not in dialects:
+                dialects[db_name] = url_obj.get_dialect()
+                dialects[db_name].load_provisioning()
             url_key = (url_obj.get_backend_name(), url_obj.host)
             urls[url_key].add(db_url)
             idents[url_key].add(db_name)

diff --git a/test/aaa_profiling/test_orm.py b/test/aaa_profiling/test_orm.py
index afa8634aa3038beeae23d95855faddf5bc63e371..061c0bff4f402ccc82e5e5d653d6f50bfd7a06d8 100644 (file)
--- a/test/aaa_profiling/test_orm.py
+++ b/test/aaa_profiling/test_orm.py
@@ -93,7 +93,7 @@ class MergeTest(fixtures.MappedTest):
 
         sess2.transaction  # autobegin
 
-        @profiling.function_call_count(variance=0.10)
+        @profiling.function_call_count(variance=0.20)
         def go1():
             return sess2.merge(p1, load=False)
 

diff --git a/test/engine/test_execute.py b/test/engine/test_execute.py
index 4f3fb20626ac917ce0d6f33a82b579c6271549e9..150038a40340247bf9bcd63e86b7045a564240ee 100644 (file)
--- a/test/engine/test_execute.py
+++ b/test/engine/test_execute.py
@@ -2838,11 +2838,14 @@ class AutocommitTextTest(fixtures.TestBase):
             options={"_initialize": False, "pool_reset_on_return": None}
         )
         engine.dialect.dbapi = dbapi
-        engine.execute("%s something table something" % keyword)
-        if expected:
-            eq_(dbapi.connect().mock_calls, [call.cursor(), call.commit()])
-        else:
-            eq_(dbapi.connect().mock_calls, [call.cursor()])
+
+        with engine.connect() as conn:
+            conn.execute("%s something table something" % keyword)
+
+            if expected:
+                eq_(dbapi.connect().mock_calls, [call.cursor(), call.commit()])
+            else:
+                eq_(dbapi.connect().mock_calls, [call.cursor()])
 
     def test_update(self):
         self._test_keyword("UPDATE")

diff --git a/test/engine/test_logging.py b/test/engine/test_logging.py
index 14b49e5e89af0b5292b815e6c09ef5c6db82dc30..fe4ff44a75e4f4da1c10eae6d4d943f3a1acf460 100644 (file)
--- a/test/engine/test_logging.py
+++ b/test/engine/test_logging.py
@@ -29,14 +29,16 @@ class LogParamsTest(fixtures.TestBase):
         self.no_param_engine = engines.testing_engine(
             options={"echo": True, "hide_parameters": True}
         )
-        self.eng.execute("create table foo (data string)")
-        self.no_param_engine.execute("create table foo (data string)")
+        self.eng.execute("create table if not exists foo (data string)")
+        self.no_param_engine.execute(
+            "create table if not exists foo (data string)"
+        )
         self.buf = logging.handlers.BufferingHandler(100)
         for log in [logging.getLogger("sqlalchemy.engine")]:
             log.addHandler(self.buf)
 
     def teardown(self):
-        self.eng.execute("drop table foo")
+        self.eng.execute("drop table if exists foo")
         for log in [logging.getLogger("sqlalchemy.engine")]:
             log.removeHandler(self.buf)
 

diff --git a/test/requirements.py b/test/requirements.py
index 532573fe526e070c2f588a621291e7109daec75c..a1bd57aea0413bd870493dc8289fa6ffd0283999 100644 (file)
--- a/test/requirements.py
+++ b/test/requirements.py
@@ -205,7 +205,9 @@ class DefaultRequirements(SuiteRequirements):
     @property
     def temporary_tables(self):
         """target database supports temporary tables"""
-        return skip_if(["mssql", "firebird"], "not supported (?)")
+        return skip_if(
+            ["mssql", "firebird", self._sqlite_file_db], "not supported (?)"
+        )
 
     @property
     def temp_table_reflection(self):
@@ -496,12 +498,14 @@ class DefaultRequirements(SuiteRequirements):
     def temp_table_names(self):
         """target dialect supports listing of temporary table names"""
 
-        return only_on(["sqlite", "oracle"])
+        return only_on(["sqlite", "oracle"]) + skip_if(self._sqlite_file_db)
 
     @property
     def temporary_views(self):
         """target database supports temporary views"""
-        return only_on(["sqlite", "postgresql"])
+        return only_on(["sqlite", "postgresql"]) + skip_if(
+            self._sqlite_file_db
+        )
 
     @property
     def update_nowait(self):
@@ -857,6 +861,14 @@ class DefaultRequirements(SuiteRequirements):
             ]
         )
 
+    def _sqlite_file_db(self, config):
+        return against(config, "sqlite") and config.db.dialect._is_url_file_db(
+            config.db.url
+        )
+
+    def _sqlite_memory_db(self, config):
+        return not self._sqlite_file_db(config)
+
     def _sqlite_json(self, config):
         if not against(config, "sqlite >= 3.9"):
             return False
@@ -1496,4 +1508,4 @@ class DefaultRequirements(SuiteRequirements):
 
     @property
     def python_profiling_backend(self):
-        return only_on(["sqlite"])
+        return only_on([self._sqlite_memory_db])

diff --git a/tox.ini b/tox.ini
index 16fbc9507c4b9a8c50927d28ce777cd0a304bfb5..9950aceb4992a4ee9e391f737201ea033a92bd78 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -63,26 +63,26 @@ setenv=
     cext: REQUIRE_SQLALCHEMY_CEXT=1
     cov: COVERAGE={[testenv]cov_args}
     sqlite: SQLITE={env:TOX_SQLITE:--db sqlite}
-    sqlite_file: BASECOMMAND=/bin/sh -c 'echo "sqlite_file backend tests not enabled for this version"'; /bin/true
+    sqlite_file: SQLITE={env:TOX_SQLITE_FILE:--db sqlite_file}
     postgresql: POSTGRESQL={env:TOX_POSTGRESQL:--db postgresql}
     mysql: MYSQL={env:TOX_MYSQL:--db mysql --db pymysql}
     oracle: ORACLE={env:TOX_ORACLE:--db oracle}
     mssql: MSSQL={env:TOX_MSSQL:--db mssql}
-    oracle,mssql: IDENTS=--write-idents db_idents.txt
-    oracle,mssql: NOMEMORY=--nomemory
+    oracle,mssql,sqlite_file: IDENTS=--write-idents db_idents.txt
+    oracle,mssql,sqlite_file: NOMEMORY=--nomemory
     backendonly: BACKENDONLY=--backend-only
 
 # tox as of 2.0 blocks all environment variables from the
 # outside, unless they are here (or in TOX_TESTENV_PASSENV,
 # wildcards OK).  Need at least these
-passenv=ORACLE_HOME NLS_LANG TOX_POSTGRESQL TOX_MYSQL TOX_ORACLE TOX_MSSQL TOX_SQLITE TOX_WORKERS
+passenv=ORACLE_HOME NLS_LANG TOX_POSTGRESQL TOX_MYSQL TOX_ORACLE TOX_MSSQL TOX_SQLITE TOX_SQLITE_FILE TOX_WORKERS
 
 # for nocext, we rm *.so in lib in case we are doing usedevelop=True
 commands=
   cext: /bin/true
   nocext: sh -c "rm -f lib/sqlalchemy/*.so"
   {env:BASECOMMAND} {env:WORKERS} {env:SQLITE:} {env:POSTGRESQL:} {env:MYSQL:} {env:ORACLE:} {env:MSSQL:} {env:BACKENDONLY:} {env:IDENTS:} {env:NOMEMORY:} {env:COVERAGE:} {posargs}
-  oracle,mssql: python reap_dbs.py db_idents.txt
+  oracle,mssql,sqlite_file: python reap_dbs.py db_idents.txt
 
 # thanks to https://julien.danjou.info/the-best-flake8-extensions/
 [testenv:pep8]