From: Shamil Abdulaev Date: Tue, 28 Apr 2026 11:23:14 +0000 (-0400) Subject: Add sqlite.JSONB type for binary JSON storage (SQLite >= 3.45.0) X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0aa47f8082513ecccc1638558f46b9fe8e610ad2;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git Add sqlite.JSONB type for binary JSON storage (SQLite >= 3.45.0) Added :class:`_sqlite.JSONB` type for SQLite's binary JSON storage format, available as of SQLite version 3.45.0. Values are stored via the ``jsonb()`` SQL function and retrieved via ``json()``, while the Python-side behavior remains identical to :class:`_sqlite.JSON`. Pull request courtesy Shamil Abdulaev. Fixes: #13260 Adds `sqlalchemy.dialects.sqlite.JSONB` — a new dialect-specific type for SQLite's binary JSON storage format, introduced in SQLite 3.45.0. The type: - renders `JSONB` in DDL (`CREATE TABLE t (col JSONB)`) - wraps bind values with `jsonb()` on write, storing data as a BLOB - wraps column reads with `json()`, returning standard text JSON to Python - reflects back from the database as `sqlite.JSONB` - behaves identically to `sqlite.JSON` on the Python side SQLite 3.45.0 introduced a native binary JSON format (JSONB) that is more compact and faster to parse than text JSON. Users who want to opt into this storage format had no way to do so via SQLAlchemy. - `JSONB` inherits from `sqlite.JSON` and defines `__visit_name__ = "JSONB"` so the DDL compiler dispatches to `visit_JSONB` instead of `visit_JSON` - Added `JSONB: JSONB` to `colspecs` so the type is not remapped to `_SQliteJson` via the `sqltypes.JSON` MRO entry - `bind_expression` / `column_expression` apply `jsonb()` / `json()` transparently; existing `result_processor` from `sqltypes.JSON` handles deserialization without changes - Added `sqlite_jsonb` requirement (checks for SQLite >= 3.45 and that `jsonb()` is available, since it is a loadable extension) `test/dialect/sqlite/test_types.py::JSONBTest` (8 tests): - DDL renders `JSONB` - Reflection returns `sqlite.JSONB` - Round-trip read/write including `None` - Sub-object extraction via `[]` indexing - Compiled SQL shows `jsonb(?)` on INSERT and `json(col)` on SELECT - `typeof(col)` returns `blob` confirming binary storage Closes: #13261 Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/13261 Pull-request-sha: 81b93af698e0222a9262614d39dd5ade6b640273 Change-Id: Ic38704674d30aa3d1bb5ce1e8ef5e4b0562ad91a --- diff --git a/doc/build/changelog/changelog_20.rst b/doc/build/changelog/changelog_20.rst index 3345d1185d..e6fa9ddcd2 100644 --- a/doc/build/changelog/changelog_20.rst +++ b/doc/build/changelog/changelog_20.rst @@ -74,7 +74,7 @@ for special types such as ``sysname``, whereas previously the base name would be received (e.g. ``nvarchar`` for ``sysname``), leading to warnings that such types could not be reflected and resulting in :class:`.NullType`, - rather than the expected :class:`.NVARCHAR` for a type like ``sysname``. + rather than the expected :class:`_mssql.NVARCHAR` for a type like ``sysname``. The column reflection query now joins ``sys.types`` a second time to look up the base type when the user type name is not present in :attr:`.MSDialect.ischema_names`, and both names are checked in @@ -799,7 +799,7 @@ :tags: usecase, postgresql :tickets: 10927 - Added support for PostgreSQL 14+ :class:`.JSONB` subscripting syntax. + Added support for PostgreSQL 14+ :class:`_postgresql.JSONB` subscripting syntax. When connected to PostgreSQL 14 or later, JSONB columns now automatically use the native subscript notation ``jsonb_col['key']`` instead of the arrow operator ``jsonb_col -> 'key'`` for both read and @@ -823,7 +823,7 @@ will resemble ``(entity.data['a'] ->> 'b')`` which will fail to produce the exact textual syntax match required by the PostgreSQL query planner. Therefore, for users upgrading to SQLAlchemy 2.0.42 - or higher, existing indexes that were created against :class:`.JSONB` + or higher, existing indexes that were created against :class:`_postgresql.JSONB` expressions that use subscripting would need to be dropped and re-created in order for them to work with the new query syntax, e.g. an expression like ``((entity.data -> 'a') ->> 'b')`` would become diff --git a/doc/build/changelog/migration_21.rst b/doc/build/changelog/migration_21.rst index 987fbb2a58..166f80f9ec 100644 --- a/doc/build/changelog/migration_21.rst +++ b/doc/build/changelog/migration_21.rst @@ -1866,3 +1866,21 @@ the :class:`.Boolean` datatype. :ref:`oracle_boolean_support` :ticket:`11633` + +SQLite +====== + +Added :class:`_sqlite.JSONB` json format for SQLite +--------------------------------------------------- + +SQLite version 3.45 added support for serializing json using +a binaly format called ``JSONB``, which provides imporved performance +and storage saving. The new :class:`_sqlite.JSONB` type provides support +for this format, ensuring that the data is correctly serialized +when inserting and deserialized when querying. + +.. seealso:: + + :class:`_sqlite.JSONB` + +:ticket:`13260` diff --git a/doc/build/changelog/unreleased_21/13260.rst b/doc/build/changelog/unreleased_21/13260.rst new file mode 100644 index 0000000000..3c49f5b6f6 --- /dev/null +++ b/doc/build/changelog/unreleased_21/13260.rst @@ -0,0 +1,13 @@ +.. change:: + :tags: feature, sqlite + :tickets: 13260 + + Added :class:`_sqlite.JSONB` type for SQLite's binary JSON storage + format, available as of SQLite version 3.45.0. Values are stored via + the ``jsonb()`` SQL function and retrieved via ``json()``, while the + Python-side behavior remains identical to :class:`_sqlite.JSON`. + Pull request courtesy Shamil Abdulaev. + + .. seealso:: + + :class:`_sqlite.JSONB` diff --git a/doc/build/dialects/mssql.rst b/doc/build/dialects/mssql.rst index 5d7d35395e..5630517c42 100644 --- a/doc/build/dialects/mssql.rst +++ b/doc/build/dialects/mssql.rst @@ -151,6 +151,8 @@ construction arguments, are as follows: :members: __init__ :noindex: +.. autoclass:: NVARCHAR + :noindex: .. autoclass:: XML :members: __init__ diff --git a/doc/build/dialects/sqlite.rst b/doc/build/dialects/sqlite.rst index d25301fa53..19cadb3c9f 100644 --- a/doc/build/dialects/sqlite.rst +++ b/doc/build/dialects/sqlite.rst @@ -38,6 +38,8 @@ they originate from :mod:`sqlalchemy.types` or from the local dialect:: .. autoclass:: JSON +.. autoclass:: JSONB + .. autoclass:: TIME SQLite DML Constructs diff --git a/lib/sqlalchemy/dialects/sqlite/__init__.py b/lib/sqlalchemy/dialects/sqlite/__init__.py index 8609082242..219047a931 100644 --- a/lib/sqlalchemy/dialects/sqlite/__init__.py +++ b/lib/sqlalchemy/dialects/sqlite/__init__.py @@ -20,6 +20,7 @@ from .base import DECIMAL from .base import FLOAT from .base import INTEGER from .base import JSON +from .base import JSONB from .base import NUMERIC from .base import REAL from .base import SMALLINT @@ -44,6 +45,7 @@ __all__ = ( "FLOAT", "INTEGER", "JSON", + "JSONB", "NUMERIC", "SMALLINT", "TEXT", diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py index 61d61d9a1e..c37bcbb374 100644 --- a/lib/sqlalchemy/dialects/sqlite/base.py +++ b/lib/sqlalchemy/dialects/sqlite/base.py @@ -995,6 +995,7 @@ from typing import Optional from typing import TYPE_CHECKING from .json import JSON +from .json import JSONB from .json import JSONIndexType from .json import JSONPathType from ... import exc @@ -1401,6 +1402,7 @@ colspecs = { sqltypes.JSON.JSONIndexType: JSONIndexType, sqltypes.JSON.JSONPathType: JSONPathType, sqltypes.Time: TIME, + JSONB: JSONB, } ischema_names = { @@ -1419,6 +1421,7 @@ ischema_names = { "INT": sqltypes.INTEGER, "INTEGER": sqltypes.INTEGER, "JSON": JSON, + "JSONB": JSONB, "NUMERIC": sqltypes.NUMERIC, "REAL": sqltypes.REAL, "SMALLINT": sqltypes.SMALLINT, @@ -1951,6 +1954,9 @@ class SQLiteTypeCompiler(compiler.GenericTypeCompiler): # numeric value. JSONTEXT can be used if this case is required. return "JSON" + def visit_JSONB(self, type_, **kw): + return "JSONB" + class SQLiteIdentifierPreparer(compiler.IdentifierPreparer): reserved_words = { diff --git a/lib/sqlalchemy/dialects/sqlite/json.py b/lib/sqlalchemy/dialects/sqlite/json.py index ac705d661d..6a4e4e8349 100644 --- a/lib/sqlalchemy/dialects/sqlite/json.py +++ b/lib/sqlalchemy/dialects/sqlite/json.py @@ -10,10 +10,12 @@ from typing import Any from typing import TYPE_CHECKING from ... import types as sqltypes +from ...sql import func from ...sql.sqltypes import _T_JSON if TYPE_CHECKING: from ...engine.interfaces import Dialect + from ...sql.elements import ColumnElement from ...sql.type_api import _BindProcessorType from ...sql.type_api import _LiteralProcessorType @@ -47,6 +49,38 @@ class JSON(sqltypes.JSON[_T_JSON]): """ +class JSONB(JSON[_T_JSON]): + """SQLite JSONB type. + + Stores JSON data in SQLite's binary JSONB format, available as of + SQLite version 3.45.0. The binary format is more compact and faster + to parse than the text-based :class:`_sqlite.JSON` type. + + Values are transparently stored using the ``jsonb()`` SQL function and + retrieved as text JSON via the ``json()`` SQL function, so the Python + side behaves identically to :class:`_sqlite.JSON`. + + .. versionadded:: 2.1b3 + + .. seealso:: + + :class:`_sqlite.JSON` + + https://sqlite.org/jsonb.html + + """ + + __visit_name__ = "JSONB" + + def bind_expression( + self, bindvalue: ColumnElement[Any] + ) -> ColumnElement[Any]: + return func.jsonb(bindvalue, type_=self) + + def column_expression(self, col: ColumnElement[Any]) -> ColumnElement[Any]: + return func.json(col, type_=self) + + # Note: these objects currently match exactly those of MySQL, however since # these are not generalizable to all JSON implementations, remain separately # implemented for each dialect. diff --git a/test/dialect/sqlite/test_types.py b/test/dialect/sqlite/test_types.py index a8068d34cb..736541da94 100644 --- a/test/dialect/sqlite/test_types.py +++ b/test/dialect/sqlite/test_types.py @@ -310,6 +310,72 @@ class JSONTest(fixtures.TestBase): eq_(jd.mock_calls, [mock.call(json.dumps(data_element))]) +class JSONBTest(fixtures.TestBase, AssertsCompiledSQL): + __requires__ = ("sqlite_jsonb",) + __only_on__ = "sqlite" + __backend__ = True + __dialect__ = "sqlite" + + def test_ddl(self, metadata, connection): + Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + result = connection.exec_driver_sql( + "select sql from sqlite_master where name='jsonb_test'" + ).scalar() + assert "JSONB" in result + + def test_reflection(self, metadata, connection): + Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + reflected = Table("jsonb_test", MetaData(), autoload_with=connection) + is_(reflected.c.foo.type._type_affinity, sqltypes.JSON) + assert isinstance(reflected.c.foo.type, sqlite.JSONB) + + def test_roundtrip(self, metadata, connection): + t = Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + value = {"json": {"foo": "bar"}, "recs": ["one", "two"]} + connection.execute(t.insert(), {"foo": value}) + eq_(connection.scalar(select(t.c.foo)), value) + + def test_roundtrip_none(self, metadata, connection): + t = Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + connection.execute(t.insert(), {"foo": None}) + eq_(connection.scalar(select(t.c.foo)), None) + + def test_extract_subobject(self, metadata, connection): + t = Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + value = {"json": {"foo": "bar"}} + connection.execute(t.insert(), {"foo": value}) + eq_(connection.scalar(select(t.c.foo["json"])), value["json"]) + + def test_bind_expression_renders_jsonb_func(self): + t = Table("jsonb_test", MetaData(), Column("foo", sqlite.JSONB)) + self.assert_compile( + t.insert().values(foo={"key": "val"}), + "INSERT INTO jsonb_test (foo) VALUES (jsonb(?))", + ) + + def test_column_expression_renders_json_func(self): + t = Table("jsonb_test", MetaData(), Column("foo", sqlite.JSONB)) + self.assert_compile( + select(t.c.foo), + "SELECT json(jsonb_test.foo) AS foo FROM jsonb_test", + ) + + def test_storage_is_binary(self, metadata, connection): + """JSONB data is stored as BLOB, not text.""" + t = Table("jsonb_test", metadata, Column("foo", sqlite.JSONB)) + metadata.create_all(connection) + connection.execute(t.insert(), {"foo": {"key": "val"}}) + raw = connection.exec_driver_sql( + "select typeof(foo) from jsonb_test" + ).scalar() + eq_(raw, "blob") + + class DateTimeTest(fixtures.TestBase, AssertsCompiledSQL): def test_time_microseconds(self): dt = datetime.datetime(2008, 6, 27, 12, 0, 0, 125) diff --git a/test/requirements.py b/test/requirements.py index 34e65f9371..df78980072 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -1313,6 +1313,10 @@ class DefaultRequirements(SuiteRequirements): except exc.DBAPIError: return False + @property + def sqlite_jsonb(self): + return only_on("sqlite >= 3.45") + @property def sqlite_memory(self): return only_on(self._sqlite_memory_db)