From: Mike Bayer Date: Fri, 20 Sep 2019 13:25:40 +0000 (-0400) Subject: Support SQLite URIs X-Git-Tag: rel_1_3_9~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2808efb01af21602ee1650c3b5d987273ee65ea0;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Support SQLite URIs Added support for sqlite "URI" connections, which allow for sqlite-specific flags to be passed in the query string such as "read only" for Python sqlite3 drivers that support this. Fixes: #4863 Change-Id: I7740b55ee8f2ede72a5c49ee94a7540e4d0250f2 (cherry picked from commit adb495503dab660f014cad0200491c854d2f6a50) --- diff --git a/doc/build/changelog/unreleased_13/4863.rst b/doc/build/changelog/unreleased_13/4863.rst new file mode 100644 index 0000000000..1fd7f4cb8b --- /dev/null +++ b/doc/build/changelog/unreleased_13/4863.rst @@ -0,0 +1,11 @@ +.. change:: + :tags: usecase, sqlite + :tickets: 4863 + + Added support for sqlite "URI" connections, which allow for sqlite-specific + flags to be passed in the query string such as "read only" for Python + sqlite3 drivers that support this. + + .. seealso:: + + :ref:`pysqlite_uri_connections` diff --git a/lib/sqlalchemy/dialects/sqlite/pysqlite.py b/lib/sqlalchemy/dialects/sqlite/pysqlite.py index c914bbfd08..4eca7ae318 100644 --- a/lib/sqlalchemy/dialects/sqlite/pysqlite.py +++ b/lib/sqlalchemy/dialects/sqlite/pysqlite.py @@ -66,6 +66,68 @@ present. Specify ``sqlite://`` and nothing else:: # in-memory database e = create_engine('sqlite://') +.. _pysqlite_uri_connections: + +URI Connections +^^^^^^^^^^^^^^^ + +Modern versions of SQLite support an alternative system of connecting using a +`driver level URI `_, which has the advantage +that additional driver-level arguments can be passed including options such as +"read only". The Python sqlite3 driver supports this mode under modern Python +3 versions. The SQLAlchemy pysqlite driver supports this mode of use by +specifing "uri=true" in the URL query string. The SQLite-level "URI" is kept +as the "database" portion of the SQLAlchemy url (that is, following a slash):: + + e = create_engine("sqlite:///file:path/to/database?mode=ro&uri=true") + +.. note:: The "uri=true" parameter must appear in the **query string** + of the URL. It will not currently work as expected if it is only + present in the :paramref:`.create_engine.connect_args` parameter dictionary. + +The logic reconciles the simultaneous presence of SQLAlchemy's query string and +SQLite's query string by separating out the parameters that belong to the +Python sqlite3 driver vs. those that belong to the SQLite URI. This is +achieved through the use of a fixed list of parameters known to be accepted by +the Python side of the driver. For example, to include a URL that indicates +the Python sqlite3 "timeout" and "check_same_thread" parameters, along with the +SQLite "mode" and "nolock" parameters, they can all be passed together on the +query string:: + + e = create_engine( + "sqlite:///file:path/to/database?" + "check_same_thread=true&timeout=10&mode=ro&nolock=1&uri=true" + ) + +Above, the pysqlite / sqlite3 DBAPI would be passed arguments as:: + + sqlite3.connect( + "file:path/to/database?mode=ro&nolock=1", + check_same_thread=True, timeout=10, uri=True + ) + +Regarding future parameters added to either the Python or native drivers. new +parameter names added to the SQLite URI scheme should be automatically +accommodated by this scheme. New parameter names added to the Python driver +side can be accommodated by specifying them in the +:paramref:`.create_engine.connect_args` dictionary, until dialect support is +added by SQLAlchemy. For the less likely case that the native SQLite driver +adds a new parameter name that overlaps with one of the existing, known Python +driver parameters (such as "timeout" perhaps), SQLAlchemy's dialect would +require adjustment for the URL scheme to continue to support this. + +As is always the case for all SQLAlchemy dialects, the entire "URL" process +can be bypassed in :func:`.create_engine` through the use of the +:paramref:`.create_engine.creator` parameter which allows for a custom callable +that creates a Python sqlite3 driver level connection directly. + +.. versionadded:: 1.3.9 + +.. seealso:: + + `Uniform Resource Identifiers `_ - in + the SQLite documentation + Compatibility with sqlite3 "native" date and datetime types ----------------------------------------------------------- @@ -317,7 +379,7 @@ class SQLiteDialect_pysqlite(SQLiteDialect): driver = "pysqlite" - def __init__(self, **kwargs): + def __init__(self, uri=False, **kwargs): SQLiteDialect.__init__(self, **kwargs) if self.dbapi is not None: @@ -362,18 +424,54 @@ class SQLiteDialect_pysqlite(SQLiteDialect): " sqlite:///relative/path/to/file.db\n" " sqlite:////absolute/path/to/file.db" % (url,) ) - filename = url.database or ":memory:" - if filename != ":memory:": - filename = os.path.abspath(filename) - - opts = url.query.copy() - util.coerce_kw_type(opts, "timeout", float) - util.coerce_kw_type(opts, "isolation_level", str) - util.coerce_kw_type(opts, "detect_types", int) - util.coerce_kw_type(opts, "check_same_thread", bool) - util.coerce_kw_type(opts, "cached_statements", int) - - return ([filename], opts) + + # theoretically, this list can be augmented, at least as far as + # parameter names accepted by sqlite3/pysqlite, using + # inspect.getfullargspec(). for the moment this seems like overkill + # as these parameters don't change very often, and as always, + # parameters passed to connect_args will always go to the + # sqlite3/pysqlite driver. + pysqlite_args = [ + ("uri", bool), + ("timeout", float), + ("isolation_level", str), + ("detect_types", int), + ("check_same_thread", bool), + ("cached_statements", int), + ] + opts = url.query + pysqlite_opts = {} + for key, type_ in pysqlite_args: + util.coerce_kw_type(opts, key, type_, dest=pysqlite_opts) + + if pysqlite_opts.get("uri", False): + uri_opts = opts.copy() + # here, we are actually separating the parameters that go to + # sqlite3/pysqlite vs. those that go the SQLite URI. What if + # two names conflict? again, this seems to be not the case right + # now, and in the case that new names are added to + # either side which overlap, again the sqlite3/pysqlite parameters + # can be passed through connect_args instead of in the URL. + # If SQLite native URIs add a parameter like "timeout" that + # we already have listed here for the python driver, then we need + # to adjust for that here. + for key, type_ in pysqlite_args: + uri_opts.pop(key, None) + filename = url.database + if uri_opts: + # sorting of keys is for unit test support + filename += "?" + ( + "&".join( + "%s=%s" % (key, uri_opts[key]) + for key in sorted(uri_opts) + ) + ) + else: + filename = url.database or ":memory:" + if filename != ":memory:": + filename = os.path.abspath(filename) + + return ([filename], pysqlite_opts) def is_disconnect(self, e, connection, cursor): return isinstance( diff --git a/lib/sqlalchemy/util/langhelpers.py b/lib/sqlalchemy/util/langhelpers.py index 9276df332b..7705972b22 100644 --- a/lib/sqlalchemy/util/langhelpers.py +++ b/lib/sqlalchemy/util/langhelpers.py @@ -1116,21 +1116,24 @@ def asint(value): return int(value) -def coerce_kw_type(kw, key, type_, flexi_bool=True): +def coerce_kw_type(kw, key, type_, flexi_bool=True, dest=None): r"""If 'key' is present in dict 'kw', coerce its value to type 'type\_' if necessary. If 'flexi_bool' is True, the string '0' is considered false when coercing to boolean. """ + if dest is None: + dest = kw + if ( key in kw and (not isinstance(type_, type) or not isinstance(kw[key], type_)) and kw[key] is not None ): if type_ is bool and flexi_bool: - kw[key] = asbool(kw[key]) + dest[key] = asbool(kw[key]) else: - kw[key] = type_(kw[key]) + dest[key] = type_(kw[key]) def constructor_copy(obj, cls, *args, **kw): diff --git a/test/dialect/test_sqlite.py b/test/dialect/test_sqlite.py index 162282a647..8694a45b81 100644 --- a/test/dialect/test_sqlite.py +++ b/test/dialect/test_sqlite.py @@ -700,13 +700,6 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): assert u("méil") in result.keys() assert ue("\u6e2c\u8a66") in result.keys() - def test_file_path_is_absolute(self): - d = pysqlite_dialect.dialect() - eq_( - d.create_connect_args(make_url("sqlite:///foo.db")), - ([os.path.abspath("foo.db")], {}), - ) - def test_pool_class(self): e = create_engine("sqlite+pysqlite://") assert e.pool.__class__ is pool.SingletonThreadPool @@ -717,6 +710,41 @@ class DialectTest(fixtures.TestBase, AssertsExecutionResults): e = create_engine("sqlite+pysqlite:///foo.db") assert e.pool.__class__ is pool.NullPool + def test_connect_args(self): + """test create_connect_args scenarios including support for uri=True""" + + d = pysqlite_dialect.dialect() + for url, expected in [ + ( + "sqlite:///foo.db", # file path is absolute + ([os.path.abspath("foo.db")], {}), + ), + ("sqlite:////abs/path/to/foo.db", (["/abs/path/to/foo.db"], {})), + ("sqlite://", ([":memory:"], {})), + ( + "sqlite:///?check_same_thread=true", + ([":memory:"], {"check_same_thread": True}), + ), + ( + "sqlite:///file:path/to/database?" + "check_same_thread=true&timeout=10&mode=ro&nolock=1&uri=true", + ( + ["file:path/to/database?mode=ro&nolock=1"], + {"check_same_thread": True, "timeout": 10.0, "uri": True}, + ), + ), + ( + "sqlite:///file:path/to/database?" "mode=ro&uri=true", + (["file:path/to/database?mode=ro"], {"uri": True}), + ), + ( + "sqlite:///file:path/to/database?uri=true", + (["file:path/to/database"], {"uri": True}), + ), + ]: + url = make_url(url) + eq_(d.create_connect_args(url), expected) + class AttachedDBTest(fixtures.TestBase): __only_on__ = "sqlite"