From: Mike Bayer Date: Mon, 3 Jul 2023 17:28:35 +0000 (-0400) Subject: add option to disable INET, CIDR result set conversion X-Git-Tag: rel_2_0_18~1^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5d709b53685d049b8c8d7a57e5319d5babcf807a;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git add option to disable INET, CIDR result set conversion Added new parameter ``native_inet_types=False`` to the all PostgreSQL dialects, which indicates the all converters used by the DBAPI to convert rows from PostgreSQL :class:`.INET` and :class:`.CIDR` columns into Python ``ipaddress`` datatypes should be disabled, returning strings instead. This allows code written to work with strings for these datatypes to be migrated to asyncpg, psycopg, or pg8000 without code changes beyond the engine parameter. Currently, some DBAPIs return ``ipaddress`` objects while others return strings for one or both of these datatypes. A future release of SQLAlchemy will attempt to normalize support for Python's ``ipaddress`` across all DBAPIs. Fixes: #9945 Change-Id: Id59e9982e2bfd6706fe335e4e700902abfb63663 --- diff --git a/doc/build/changelog/unreleased_20/9945.rst b/doc/build/changelog/unreleased_20/9945.rst new file mode 100644 index 0000000000..c059ca581a --- /dev/null +++ b/doc/build/changelog/unreleased_20/9945.rst @@ -0,0 +1,16 @@ +.. change:: + :tags: bug, postgresql + :tickets: 9945 + + Added new parameter ``native_inet_types=False`` to all PostgreSQL + dialects, which indicates converters used by the DBAPI to + convert rows from PostgreSQL :class:`.INET` and :class:`.CIDR` columns + into Python ``ipaddress`` datatypes should be disabled, returning strings + instead. This allows code written to work with strings for these datatypes + to be migrated to asyncpg, psycopg, or pg8000 without code changes + other than adding this parameter to the :func:`_sa.create_engine` + or :func:`_asyncio.create_async_engine` function call. + + .. seealso:: + + :ref:`postgresql_network_datatypes` diff --git a/doc/build/dialects/postgresql.rst b/doc/build/dialects/postgresql.rst index 71a017ee06..3e981f04aa 100644 --- a/doc/build/dialects/postgresql.rst +++ b/doc/build/dialects/postgresql.rst @@ -304,7 +304,47 @@ The available multirange datatypes are as follows: * :class:`_postgresql.TSMULTIRANGE` * :class:`_postgresql.TSTZMULTIRANGE` +.. _postgresql_network_datatypes: + +Network Data Types +------------------ + +The included networking datatypes are :class:`_postgresql.INET`, +:class:`_postgresql.CIDR`, :class:`_postgresql.MACADDR`. + +For :class:`_postgresql.INET` and :class:`_postgresql.CIDR` datatypes, +conditional support is available for these datatypes to send and retrieve +Python ``ipaddress`` objects including ``ipaddress.IPv4Network``, +``ipaddress.IPv6Network``, ``ipaddress.IPv4Address``, +``ipaddress.IPv6Address``. This support is currently **the default behavior of +the DBAPI itself, and varies per DBAPI. SQLAlchemy does not yet implement its +own network address conversion logic**. + +* The :ref:`postgresql_psycopg` and :ref:`postgresql_asyncpg` support these + datatypes fully; objects from the ``ipaddress`` family are returned in rows + by default. +* The :ref:`postgresql_psycopg2` dialect only sends and receives strings. +* The :ref:`postgresql_pg8000` dialect supports ``ipaddress.IPv4Address`` and + ``ipaddress.IPv6Address`` objects for the :class:`_postgresql.INET` datatype, + but uses strings for :class:`_postgresql.CIDR` types. + +To **normalize all the above DBAPIs to only return strings**, use the +``native_inet_types`` parameter, passing a value of ``False``:: + + e = create_engine( + "postgresql+psycopg://scott:tiger@host/dbname", native_inet_types=False + ) + +With the above parameter, the ``psycopg``, ``asyncpg`` and ``pg8000`` dialects +will disable the DBAPI's adaptation of these types and will return only strings, +matching the behavior of the older ``psycopg2`` dialect. + +The parameter may also be set to ``True``, where it will have the effect of +raising ``NotImplementedError`` for those backends that don't support, or +don't yet fully support, conversion of rows to Python ``ipaddress`` datatypes +(currently psycopg2 and pg8000). +.. versionadded:: 2.0.18 - added the ``native_inet_types`` parameter. PostgreSQL Data Types --------------------- @@ -553,6 +593,8 @@ psycopg .. automodule:: sqlalchemy.dialects.postgresql.psycopg +.. _postgresql_pg8000: + pg8000 ------ @@ -560,6 +602,8 @@ pg8000 .. _dialect-postgresql-asyncpg: +.. _postgresql_asyncpg: + asyncpg ------- diff --git a/lib/sqlalchemy/dialects/postgresql/asyncpg.py b/lib/sqlalchemy/dialects/postgresql/asyncpg.py index dacb9ebd5a..e3713c6d6e 100644 --- a/lib/sqlalchemy/dialects/postgresql/asyncpg.py +++ b/lib/sqlalchemy/dialects/postgresql/asyncpg.py @@ -1184,6 +1184,25 @@ class PGDialect_asyncpg(PGDialect): format="binary", ) + async def _disable_asyncpg_inet_codecs(self, conn): + asyncpg_connection = conn._connection + + await asyncpg_connection.set_type_codec( + "inet", + encoder=lambda s: s, + decoder=lambda s: s, + schema="pg_catalog", + format="text", + ) + + await asyncpg_connection.set_type_codec( + "cidr", + encoder=lambda s: s, + decoder=lambda s: s, + schema="pg_catalog", + format="text", + ) + def on_connect(self): """on_connect for asyncpg @@ -1200,6 +1219,9 @@ class PGDialect_asyncpg(PGDialect): def connect(conn): conn.await_(self.setup_asyncpg_json_codec(conn)) conn.await_(self.setup_asyncpg_jsonb_codec(conn)) + + if self._native_inet_types is False: + conn.await_(self._disable_asyncpg_inet_codecs(conn)) if super_connect is not None: super_connect(conn) diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 1dcf58cecb..2d5f5c5ac9 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -3027,9 +3027,16 @@ class PGDialect(default.DefaultDialect): _supports_create_index_concurrently = True _supports_drop_index_concurrently = True - def __init__(self, json_serializer=None, json_deserializer=None, **kwargs): + def __init__( + self, + native_inet_types=None, + json_serializer=None, + json_deserializer=None, + **kwargs, + ): default.DefaultDialect.__init__(self, **kwargs) + self._native_inet_types = native_inet_types self._json_deserializer = json_deserializer self._json_serializer = json_serializer diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index 7536f89e59..e00628fbf2 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -470,6 +470,13 @@ class PGDialect_pg8000(PGDialect): if self._dbapi_version < (1, 16, 6): raise NotImplementedError("pg8000 1.16.6 or greater is required") + if self._native_inet_types: + raise NotImplementedError( + "The pg8000 dialect does not fully implement " + "ipaddress type handling; INET is supported by default, " + "CIDR is not" + ) + @util.memoized_property def _dbapi_version(self): if self.dbapi and hasattr(self.dbapi, "__version__"): @@ -615,6 +622,17 @@ class PGDialect_pg8000(PGDialect): fns.append(on_connect) + if self._native_inet_types is False: + + def on_connect(conn): + # inet + conn.register_in_adapter(869, lambda s: s) + + # cidr + conn.register_in_adapter(650, lambda s: s) + + fns.append(on_connect) + if self._json_deserializer: def on_connect(conn): diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg.py b/lib/sqlalchemy/dialects/postgresql/psycopg.py index 5c58daa3e7..f4a0fc0aa7 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg.py @@ -310,6 +310,16 @@ class PGDialect_psycopg(_PGDialect_common_psycopg): self.dbapi.adapters ) + if self._native_inet_types is False: + import psycopg.types.string + + adapters_map.register_loader( + "inet", psycopg.types.string.TextLoader + ) + adapters_map.register_loader( + "cidr", psycopg.types.string.TextLoader + ) + if self._json_deserializer: from psycopg.types.json import set_json_loads diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index 112e9578e7..af1a33be06 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -627,6 +627,13 @@ class PGDialect_psycopg2(_PGDialect_common_psycopg): ): _PGDialect_common_psycopg.__init__(self, **kwargs) + if self._native_inet_types: + raise NotImplementedError( + "The psycopg2 dialect does not implement " + "ipaddress type handling; native_inet_types cannot be set " + "to ``True`` when using this dialect." + ) + # Parse executemany_mode argument, allowing it to be only one of the # symbol names self.executemany_mode = parse_user_argument_for_enum( diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index ab417e8813..269d0e8082 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -1,6 +1,10 @@ import datetime import decimal from enum import Enum as _PY_Enum +from ipaddress import IPv4Address +from ipaddress import IPv4Network +from ipaddress import IPv6Address +from ipaddress import IPv6Network import re import uuid @@ -5964,3 +5968,127 @@ class CITextTest(fixtures.TablesTest): ).scalar() assert ret is not None + + +class InetRoundTripTests(fixtures.TestBase): + __backend__ = True + __only_on__ = "postgresql" + + def _combinations(): + return testing.combinations( + ( + postgresql.INET, + lambda: [ + "1.1.1.1", + "192.168.1.1", + "10.1.2.25", + "192.168.22.5", + ], + IPv4Address, + ), + ( + postgresql.INET, + lambda: [ + "2001:db8::1000", + ], + IPv6Address, + ), + ( + postgresql.CIDR, + lambda: [ + "10.0.0.0/8", + "192.168.1.0/24", + "192.168.0.0/16", + "192.168.1.25/32", + ], + IPv4Network, + ), + ( + postgresql.CIDR, + lambda: [ + "::ffff:1.2.3.0/120", + ], + IPv6Network, + ), + argnames="datatype,values,pytype", + ) + + @_combinations() + def test_default_native_inet_types( + self, datatype, values, pytype, connection, metadata + ): + t = Table( + "t", + metadata, + Column("id", Integer, primary_key=True), + Column("data", datatype), + ) + metadata.create_all(connection) + + connection.execute( + t.insert(), + [ + {"id": i, "data": val} + for i, val in enumerate(values(), start=1) + ], + ) + + if testing.against(["+psycopg", "+asyncpg"]) or ( + testing.against("+pg8000") + and issubclass(datatype, postgresql.INET) + ): + eq_( + connection.scalars(select(t.c.data).order_by(t.c.id)).all(), + [pytype(val) for val in values()], + ) + else: + eq_( + connection.scalars(select(t.c.data).order_by(t.c.id)).all(), + values(), + ) + + @_combinations() + def test_str_based_inet_handlers( + self, datatype, values, pytype, testing_engine, metadata + ): + t = Table( + "t", + metadata, + Column("id", Integer, primary_key=True), + Column("data", datatype), + ) + + e = testing_engine(options={"native_inet_types": False}) + with e.begin() as connection: + metadata.create_all(connection) + + connection.execute( + t.insert(), + [ + {"id": i, "data": val} + for i, val in enumerate(values(), start=1) + ], + ) + + with e.connect() as connection: + eq_( + connection.scalars(select(t.c.data).order_by(t.c.id)).all(), + values(), + ) + + @testing.only_on("+psycopg2") + def test_not_impl_psycopg2(self, testing_engine): + with expect_raises_message( + NotImplementedError, + "The psycopg2 dialect does not implement ipaddress type handling", + ): + testing_engine(options={"native_inet_types": True}) + + @testing.only_on("+pg8000") + def test_not_impl_pg8000(self, testing_engine): + with expect_raises_message( + NotImplementedError, + "The pg8000 dialect does not fully implement " + "ipaddress type handling", + ): + testing_engine(options={"native_inet_types": True})