From: Mike Bayer Date: Sun, 1 Feb 2015 00:04:54 +0000 (-0500) Subject: - Added support for the :class:`postgresql.JSONB` datatype when X-Git-Tag: rel_1_0_0b1~82 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bf70f556b382dc376783efbcb598e0fab71ee235;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git - Added support for the :class:`postgresql.JSONB` datatype when using psycopg2 2.5.4 or greater, which features native conversion of JSONB data so that SQLAlchemy's converters must be disabled; additionally, the newly added psycopg2 extension ``extras.register_default_jsonb`` is used to establish a JSON deserializer passed to the dialect via the ``json_deserializer`` argument. Also repaired the Postgresql integration tests which weren't actually round-tripping the JSONB type as opposed to the JSON type. Pull request courtesy Mateusz Susik. - Repaired the use of the "array_oid" flag when registering the HSTORE type with older psycopg2 versions < 2.4.3, which does not support this flag, as well as use of the native json serializer hook "register_default_json" with user-defined ``json_deserializer`` on psycopg2 versions < 2.5, which does not include native json. --- diff --git a/README.unittests.rst b/README.unittests.rst index 7ca229df60..5aeba51f6c 100644 --- a/README.unittests.rst +++ b/README.unittests.rst @@ -198,6 +198,18 @@ expect them to be present will fail. Additional steps specific to individual databases are as follows:: + POSTGRESQL: To enable unicode testing with JSONB, create the + database with UTF8 encoding:: + + postgres=# create database test with owner=scott encoding='utf8' template=template0; + + To include tests for HSTORE, create the HSTORE type engine:: + + postgres=# \c test; + You are now connected to database "test" as user "postgresql". + test=# create extension hstore; + CREATE EXTENSION + MYSQL: Default storage engine should be "MyISAM". Tests that require "InnoDB" as the engine will specify this explicitly. diff --git a/doc/build/changelog/changelog_09.rst b/doc/build/changelog/changelog_09.rst index 8d4c8d7f67..2af1cd35f0 100644 --- a/doc/build/changelog/changelog_09.rst +++ b/doc/build/changelog/changelog_09.rst @@ -14,6 +14,29 @@ .. changelog:: :version: 0.9.9 + .. change:: + :tags: bug, postgresql + :pullreq: github:145 + + Added support for the :class:`postgresql.JSONB` datatype when + using psycopg2 2.5.4 or greater, which features native conversion + of JSONB data so that SQLAlchemy's converters must be disabled; + additionally, the newly added psycopg2 extension + ``extras.register_default_jsonb`` is used to establish a JSON + deserializer passed to the dialect via the ``json_deserializer`` + argument. Also repaired the Postgresql integration tests which + weren't actually round-tripping the JSONB type as opposed to the + JSON type. Pull request courtesy Mateusz Susik. + + .. change:: + :tags: bug, postgresql + + Repaired the use of the "array_oid" flag when registering the + HSTORE type with older psycopg2 versions < 2.4.3, which does not + support this flag, as well as use of the native json serializer + hook "register_default_json" with user-defined ``json_deserializer`` + on psycopg2 versions < 2.5, which does not include native json. + .. change:: :tags: bug, schema :tickets: 3298, 1765 diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index 21011b68bc..26e45fed2e 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -263,14 +263,17 @@ HSTORE type The ``psycopg2`` DBAPI includes an extension to natively handle marshalling of the HSTORE type. The SQLAlchemy psycopg2 dialect will enable this extension -by default when it is detected that the target database has the HSTORE -type set up for use. In other words, when the dialect makes the first +by default when psycopg2 version 2.4 or greater is used, and +it is detected that the target database has the HSTORE type set up for use. +In other words, when the dialect makes the first connection, a sequence like the following is performed: 1. Request the available HSTORE oids using ``psycopg2.extras.HstoreAdapter.get_oids()``. If this function returns a list of HSTORE identifiers, we then determine that the ``HSTORE`` extension is present. + This function is **skipped** if the version of psycopg2 installed is + less than version 2.4. 2. If the ``use_native_hstore`` flag is at its default of ``True``, and we've detected that ``HSTORE`` oids are available, the @@ -583,19 +586,22 @@ class PGDialect_psycopg2(PGDialect): hstore_oids = self._hstore_oids(conn) if hstore_oids is not None: oid, array_oid = hstore_oids + kw = {'oid': oid} if util.py2k: - extras.register_hstore(conn, oid=oid, - array_oid=array_oid, - unicode=True) - else: - extras.register_hstore(conn, oid=oid, - array_oid=array_oid) + kw['unicode'] = True + if self.psycopg2_version >= (2, 4, 3): + kw['array_oid'] = array_oid + extras.register_hstore(conn, **kw) fns.append(on_connect) if self.dbapi and self._json_deserializer: def on_connect(conn): - extras.register_default_json( - conn, loads=self._json_deserializer) + if self._has_native_json: + extras.register_default_json( + conn, loads=self._json_deserializer) + if self._has_native_jsonb: + extras.register_default_jsonb( + conn, loads=self._json_deserializer) fns.append(on_connect) if fns: diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 1f572c9a1e..5e00fd6054 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -1948,13 +1948,15 @@ class JSONRoundTripTest(fixtures.TablesTest): __only_on__ = ('postgresql >= 9.3',) __backend__ = True + test_type = JSON + @classmethod def define_tables(cls, metadata): Table('data_table', metadata, Column('id', Integer, primary_key=True), Column('name', String(30), nullable=False), - Column('data', JSON), - Column('nulldata', JSON(none_as_null=True)) + Column('data', cls.test_type), + Column('nulldata', cls.test_type(none_as_null=True)) ) def _fixture_data(self, engine): @@ -2016,7 +2018,8 @@ class JSONRoundTripTest(fixtures.TablesTest): else: options = {} - if testing.against("postgresql+psycopg2"): + if testing.against("postgresql+psycopg2") and \ + testing.db.dialect.psycopg2_version >= (2, 5): from psycopg2.extras import register_default_json engine = engines.testing_engine(options=options) @@ -2037,7 +2040,7 @@ class JSONRoundTripTest(fixtures.TablesTest): def test_reflect(self): insp = inspect(testing.db) cols = insp.get_columns('data_table') - assert isinstance(cols[2]['type'], JSON) + assert isinstance(cols[2]['type'], self.test_type) @testing.only_on("postgresql+psycopg2") def test_insert_native(self): @@ -2096,7 +2099,7 @@ class JSONRoundTripTest(fixtures.TablesTest): "key": "value", "x": "q" }, - JSON + self.test_type ) ]) eq_( @@ -2172,7 +2175,7 @@ class JSONRoundTripTest(fixtures.TablesTest): "key": "value", "key2": {"k1": "v1", "k2": "v2"} }, - JSON + self.test_type ) ]) eq_( @@ -2199,7 +2202,7 @@ class JSONRoundTripTest(fixtures.TablesTest): util.u('réveillé'): util.u('réveillé'), "data": {"k1": util.u('drôle')} }, - JSON + self.test_type ) ]) eq_( @@ -2266,3 +2269,13 @@ class JSONBTest(JSONTest): class JSONBRoundTripTest(JSONRoundTripTest): __only_on__ = ('postgresql >= 9.4',) + + test_type = JSONB + + @testing.requires.postgresql_utf8_server_encoding + def test_unicode_round_trip_python(self): + super(JSONBRoundTripTest, self).test_unicode_round_trip_python() + + @testing.requires.postgresql_utf8_server_encoding + def test_unicode_round_trip_native(self): + super(JSONBRoundTripTest, self).test_unicode_round_trip_native() diff --git a/test/requirements.py b/test/requirements.py index 89fc108b96..4d5869226f 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -816,3 +816,9 @@ class DefaultRequirements(SuiteRequirements): return against(config, 'mysql') and \ config.db.dialect._detect_casing(config.db) == 0 + @property + def postgresql_utf8_server_encoding(self): + return only_if( + lambda config: against(config, 'postgresql') and + config.db.scalar("show server_encoding").lower() == "utf8" + )