From: Mike Bayer Date: Fri, 5 Aug 2022 17:17:44 +0000 (-0400) Subject: credit @zeeeeeb for PG Multirange patch X-Git-Tag: rel_2_0_0b1~126 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8cb6fbfb6589078484f29059dab6bd3e3646f938;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git credit @zeeeeeb for PG Multirange patch Also move some of the PG docs to the .rst page so we can link to sections. References: #7156 Change-Id: If57abc768d4768058ffa768f9bf72f83c1ee6c29 --- diff --git a/doc/build/changelog/unreleased_20/7156.rst b/doc/build/changelog/unreleased_20/7156.rst index e54d716899..2d409521f8 100644 --- a/doc/build/changelog/unreleased_20/7156.rst +++ b/doc/build/changelog/unreleased_20/7156.rst @@ -2,16 +2,16 @@ :tags: postgresql, usecase :tickets: 7156 - Adds support for PostgreSQL MultiRange types, introduced in PostgreSQL 14. - This integrates with both the ``psycopg.types.range`` extension module - as well as new the :class:`_postgresql.MultiRange` datatype introduced - in SQLAlchemy 2.0 for all PostgreSQL backends. + Adds support for PostgreSQL multirange types, introduced in PostgreSQL 14. + Support for PostgreSQL ranges and multiranges has now been generalized to + the psycopg3, psycopg2 and asyncpg backends, with room for further dialect + support, using a backend-agnostic :class:`_postgresql.Range` data object + that's constructor-compatible with the previously used psycopg2 object. See + the new documentation for usage patterns. + Thanks very much to @zeeeeeb for the pull request implementing and testing + the new datatypes and psycopg support. -.. change:: - :tags: postgresql, feature + .. seealso:: - Implemented support for PostgreSQL ranges and multiranges for all - PostgreSQL backends. To establish range and multirange types, use the - new :class:`_postgresql.Range` and :class:`_postgresql.MultiRange` - datatypes. \ No newline at end of file + :ref:`postgresql_ranges` \ No newline at end of file diff --git a/doc/build/dialects/postgresql.rst b/doc/build/dialects/postgresql.rst index a20c515e95..9ec6ee9612 100644 --- a/doc/build/dialects/postgresql.rst +++ b/doc/build/dialects/postgresql.rst @@ -5,6 +5,288 @@ PostgreSQL .. automodule:: sqlalchemy.dialects.postgresql.base +ARRAY Types +----------- + +The PostgreSQL dialect supports arrays, both as multidimensional column types +as well as array literals: + +* :class:`_postgresql.ARRAY` - ARRAY datatype + +* :class:`_postgresql.array` - array literal + +* :func:`_postgresql.array_agg` - ARRAY_AGG SQL function + +* :class:`_postgresql.aggregate_order_by` - helper for PG's ORDER BY aggregate + function syntax. + +JSON Types +---------- + +The PostgreSQL dialect supports both JSON and JSONB datatypes, including +psycopg2's native support and support for all of PostgreSQL's special +operators: + +* :class:`_postgresql.JSON` + +* :class:`_postgresql.JSONB` + +HSTORE Type +----------- + +The PostgreSQL HSTORE type as well as hstore literals are supported: + +* :class:`_postgresql.HSTORE` - HSTORE datatype + +* :class:`_postgresql.hstore` - hstore literal + +ENUM Types +---------- + +PostgreSQL has an independently creatable TYPE structure which is used +to implement an enumerated type. This approach introduces significant +complexity on the SQLAlchemy side in terms of when this type should be +CREATED and DROPPED. The type object is also an independently reflectable +entity. The following sections should be consulted: + +* :class:`_postgresql.ENUM` - DDL and typing support for ENUM. + +* :meth:`.PGInspector.get_enums` - retrieve a listing of current ENUM types + +* :meth:`.postgresql.ENUM.create` , :meth:`.postgresql.ENUM.drop` - individual + CREATE and DROP commands for ENUM. + +.. _postgresql_array_of_enum: + +Using ENUM with ARRAY +^^^^^^^^^^^^^^^^^^^^^ + +The combination of ENUM and ARRAY is not directly supported by backend +DBAPIs at this time. Prior to SQLAlchemy 1.3.17, a special workaround +was needed in order to allow this combination to work, described below. + +.. versionchanged:: 1.3.17 The combination of ENUM and ARRAY is now directly + handled by SQLAlchemy's implementation without any workarounds needed. + +.. sourcecode:: python + + from sqlalchemy import TypeDecorator + from sqlalchemy.dialects.postgresql import ARRAY + + class ArrayOfEnum(TypeDecorator): + impl = ARRAY + + def bind_expression(self, bindvalue): + return sa.cast(bindvalue, self) + + def result_processor(self, dialect, coltype): + super_rp = super(ArrayOfEnum, self).result_processor( + dialect, coltype) + + def handle_raw_string(value): + inner = re.match(r"^{(.*)}$", value).group(1) + return inner.split(",") if inner else [] + + def process(value): + if value is None: + return None + return super_rp(handle_raw_string(value)) + return process + +E.g.:: + + Table( + 'mydata', metadata, + Column('id', Integer, primary_key=True), + Column('data', ArrayOfEnum(ENUM('a', 'b, 'c', name='myenum'))) + + ) + +This type is not included as a built-in type as it would be incompatible +with a DBAPI that suddenly decides to support ARRAY of ENUM directly in +a new version. + +.. _postgresql_array_of_json: + +Using JSON/JSONB with ARRAY +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Similar to using ENUM, prior to SQLAlchemy 1.3.17, for an ARRAY of JSON/JSONB +we need to render the appropriate CAST. Current psycopg2 drivers accommodate +the result set correctly without any special steps. + +.. versionchanged:: 1.3.17 The combination of JSON/JSONB and ARRAY is now + directly handled by SQLAlchemy's implementation without any workarounds + needed. + +.. sourcecode:: python + + class CastingArray(ARRAY): + def bind_expression(self, bindvalue): + return sa.cast(bindvalue, self) + +E.g.:: + + Table( + 'mydata', metadata, + Column('id', Integer, primary_key=True), + Column('data', CastingArray(JSONB)) + ) + +.. _postgresql_ranges: + +Range and Multirange Types +-------------------------- + +PostgreSQL range and multirange types are supported for the psycopg2, +psycopg, and asyncpg dialects. + +Data values being passed to the database may be passed as string +values or by using the :class:`_postgresql.Range` data object. + +.. versionadded:: 2.0 Added the backend-agnostic :class:`_postgresql.Range` + object used to indicate ranges. The ``psycopg2``-specific range classes + are no longer exposed and are only used internally by that particular + dialect. + +E.g. an example of a fully typed model using the +:class:`_postgresql.TSRANGE` datatype:: + + from datetime import datetime + + from sqlalchemy.dialects.postgresql import Range + from sqlalchemy.dialects.postgresql import TSRANGE + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_column + + class Base(DeclarativeBase): + pass + + class RoomBooking(Base): + + __tablename__ = 'room_booking' + + id: Mapped[int] = mapped_column(primary_key=True) + room: Mapped[str] + during: Mapped[Range[datetime]] = mapped_column(TSRANGE) + +To represent data for the ``during`` column above, the :class:`_postgresql.Range` +type is a simple dataclass that will represent the bounds of the range. +Below illustrates an INSERT of a row into the above ``room_booking`` table:: + + from sqlalchemy import create_engine + from sqlalchemy.orm import Session + + engine = create_engine("postgresql+psycopg://scott:tiger@pg14/dbname") + + Base.metadata.create_all(engine) + + with Session(engine) as session: + booking = RoomBooking( + room="101", + during=Range(datetime(2013, 3, 23), datetime(2013, 3, 25)) + ) + session.add(booking) + session.commit() + +Selecting from any range column will also return :class:`_postgresql.Range` +objects as indicated:: + + from sqlalchemy import select + + with Session(engine) as session: + for row in session.execute(select(RoomBooking.during)): + print(row) + +The available range datatypes are as follows: + +* :class:`_postgresql.INT4RANGE` +* :class:`_postgresql.INT8RANGE` +* :class:`_postgresql.NUMRANGE` +* :class:`_postgresql.DATERANGE` +* :class:`_postgresql.TSRANGE` +* :class:`_postgresql.TSTZRANGE` + +.. autoclass:: sqlalchemy.dialects.postgresql.Range + +Multiranges +^^^^^^^^^^^ + +Multiranges are supported by PostgreSQL 14 and above. SQLAlchemy's +multirange datatypes deal in lists of :class:`_postgresql.Range` types. + +.. versionadded:: 2.0 Added support for MULTIRANGE datatypes. In contrast + to the ``psycopg`` multirange feature, SQLAlchemy's adaptation represents + a multirange datatype as a list of :class:`_postgresql.Range` objects. + +The example below illustrates use of the :class:`_postgresql.TSMULTIRANGE` +datatype:: + + from datetime import datetime + from typing import List + + from sqlalchemy.dialects.postgresql import Range + from sqlalchemy.dialects.postgresql import TSMULTIRANGE + from sqlalchemy.orm import DeclarativeBase + from sqlalchemy.orm import Mapped + from sqlalchemy.orm import mapped_column + + class Base(DeclarativeBase): + pass + + class EventCalendar(Base): + + __tablename__ = 'event_calendar' + + id: Mapped[int] = mapped_column(primary_key=True) + event_name: Mapped[str] + in_session_periods: Mapped[List[Range[datetime]]] = mapped_column(TSMULTIRANGE) + +Illustrating insertion and selecting of a record:: + + from sqlalchemy import create_engine + from sqlalchemy import select + from sqlalchemy.orm import Session + + engine = create_engine("postgresql+psycopg://scott:tiger@pg14/test") + + Base.metadata.create_all(engine) + + with Session(engine) as session: + calendar = EventCalendar( + event_name="SQLAlchemy Tutorial Sessions", + in_session_periods= [ + Range(datetime(2013, 3, 23), datetime(2013, 3, 25)), + Range(datetime(2013, 4, 12), datetime(2013, 4, 15)), + Range(datetime(2013, 5, 9), datetime(2013, 5, 12)), + ] + ) + session.add(calendar) + session.commit() + + for multirange in session.scalars(select(EventCalendar.in_session_periods)): + for range_ in multirange: + print(f"Start: {range_.lower} End: {range_.upper}") + +.. note:: In the above example, the list of :class:`_postgresql.Range` types + as handled by the ORM will not automatically detect in-place changes to + a particular list value; to update list values with the ORM, either re-assign + a new list to the attribute, or use the :class:`.MutableList` + type modifier. See the section :ref:`mutable_toplevel` for background. + + +The available multirange datatypes are as follows: + +* :class:`_postgresql.INT4MULTIRANGE` +* :class:`_postgresql.INT8MULTIRANGE` +* :class:`_postgresql.NUMMULTIRANGE` +* :class:`_postgresql.DATEMULTIRANGE` +* :class:`_postgresql.TSMULTIRANGE` +* :class:`_postgresql.TSTZMULTIRANGE` + + + PostgreSQL Data Types --------------------- diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 2ee679e8ea..dcd03e6257 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -1317,285 +1317,6 @@ itself:: .. versionadded:: 1.4.0b2 -ARRAY Types ------------ - -The PostgreSQL dialect supports arrays, both as multidimensional column types -as well as array literals: - -* :class:`_postgresql.ARRAY` - ARRAY datatype - -* :class:`_postgresql.array` - array literal - -* :func:`_postgresql.array_agg` - ARRAY_AGG SQL function - -* :class:`_postgresql.aggregate_order_by` - helper for PG's ORDER BY aggregate - function syntax. - -JSON Types ----------- - -The PostgreSQL dialect supports both JSON and JSONB datatypes, including -psycopg2's native support and support for all of PostgreSQL's special -operators: - -* :class:`_postgresql.JSON` - -* :class:`_postgresql.JSONB` - -HSTORE Type ------------ - -The PostgreSQL HSTORE type as well as hstore literals are supported: - -* :class:`_postgresql.HSTORE` - HSTORE datatype - -* :class:`_postgresql.hstore` - hstore literal - -ENUM Types ----------- - -PostgreSQL has an independently creatable TYPE structure which is used -to implement an enumerated type. This approach introduces significant -complexity on the SQLAlchemy side in terms of when this type should be -CREATED and DROPPED. The type object is also an independently reflectable -entity. The following sections should be consulted: - -* :class:`_postgresql.ENUM` - DDL and typing support for ENUM. - -* :meth:`.PGInspector.get_enums` - retrieve a listing of current ENUM types - -* :meth:`.postgresql.ENUM.create` , :meth:`.postgresql.ENUM.drop` - individual - CREATE and DROP commands for ENUM. - -.. _postgresql_array_of_enum: - -Using ENUM with ARRAY -^^^^^^^^^^^^^^^^^^^^^ - -The combination of ENUM and ARRAY is not directly supported by backend -DBAPIs at this time. Prior to SQLAlchemy 1.3.17, a special workaround -was needed in order to allow this combination to work, described below. - -.. versionchanged:: 1.3.17 The combination of ENUM and ARRAY is now directly - handled by SQLAlchemy's implementation without any workarounds needed. - -.. sourcecode:: python - - from sqlalchemy import TypeDecorator - from sqlalchemy.dialects.postgresql import ARRAY - - class ArrayOfEnum(TypeDecorator): - impl = ARRAY - - def bind_expression(self, bindvalue): - return sa.cast(bindvalue, self) - - def result_processor(self, dialect, coltype): - super_rp = super(ArrayOfEnum, self).result_processor( - dialect, coltype) - - def handle_raw_string(value): - inner = re.match(r"^{(.*)}$", value).group(1) - return inner.split(",") if inner else [] - - def process(value): - if value is None: - return None - return super_rp(handle_raw_string(value)) - return process - -E.g.:: - - Table( - 'mydata', metadata, - Column('id', Integer, primary_key=True), - Column('data', ArrayOfEnum(ENUM('a', 'b, 'c', name='myenum'))) - - ) - -This type is not included as a built-in type as it would be incompatible -with a DBAPI that suddenly decides to support ARRAY of ENUM directly in -a new version. - -.. _postgresql_array_of_json: - -Using JSON/JSONB with ARRAY -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Similar to using ENUM, prior to SQLAlchemy 1.3.17, for an ARRAY of JSON/JSONB -we need to render the appropriate CAST. Current psycopg2 drivers accommodate -the result set correctly without any special steps. - -.. versionchanged:: 1.3.17 The combination of JSON/JSONB and ARRAY is now - directly handled by SQLAlchemy's implementation without any workarounds - needed. - -.. sourcecode:: python - - class CastingArray(ARRAY): - def bind_expression(self, bindvalue): - return sa.cast(bindvalue, self) - -E.g.:: - - Table( - 'mydata', metadata, - Column('id', Integer, primary_key=True), - Column('data', CastingArray(JSONB)) - ) - -Range and Multirange Types --------------------------- - -PostgreSQL range and multirange types are supported for the psycopg2, -psycopg, and asyncpg dialects. - -Data values being passed to the database may be passed as string -values or by using the :class:`_postgresql.Range` data object. - -.. versionadded:: 2.0 Added the backend-agnostic :class:`_postgresql.Range` - object used to indicate ranges. The ``psycopg2``-specific range classes - are no longer exposed and are only used internally by that particular - dialect. - -E.g. an example of a fully typed model using the -:class:`_postgresql.TSRANGE` datatype:: - - from datetime import datetime - - from sqlalchemy.dialects.postgresql import Range - from sqlalchemy.dialects.postgresql import TSRANGE - from sqlalchemy.orm import DeclarativeBase - from sqlalchemy.orm import Mapped - from sqlalchemy.orm import mapped_column - - class Base(DeclarativeBase): - pass - - class RoomBooking(Base): - - __tablename__ = 'room_booking' - - id: Mapped[int] = mapped_column(primary_key=True) - room: Mapped[str] - during: Mapped[Range[datetime]] = mapped_column(TSRANGE) - -To represent data for the ``during`` column above, the :class:`_postgresql.Range` -type is a simple dataclass that will represent the bounds of the range. -Below illustrates an INSERT of a row into the above ``room_booking`` table:: - - from sqlalchemy import create_engine - from sqlalchemy.orm import Session - - engine = create_engine("postgresql+psycopg://scott:tiger@pg14/dbname") - - Base.metadata.create_all(engine) - - with Session(engine) as session: - booking = RoomBooking( - room="101", - during=Range(datetime(2013, 3, 23), datetime(2013, 3, 25)) - ) - session.add(booking) - session.commit() - -Selecting from any range column will also return :class:`_postgresql.Range` -objects as indicated:: - - from sqlalchemy import select - - with Session(engine) as session: - for row in session.execute(select(RoomBooking.during)): - print(row) - -The available range datatypes are as follows: - -* :class:`_postgresql.INT4RANGE` -* :class:`_postgresql.INT8RANGE` -* :class:`_postgresql.NUMRANGE` -* :class:`_postgresql.DATERANGE` -* :class:`_postgresql.TSRANGE` -* :class:`_postgresql.TSTZRANGE` - -.. autoclass:: sqlalchemy.dialects.postgresql.Range - -Multiranges -^^^^^^^^^^^ - -Multiranges are supported by PostgreSQL 14 and above. SQLAlchemy's -multirange datatypes deal in lists of :class:`_postgresql.Range` types. - -.. versionadded:: 2.0 Added support for MULTIRANGE datatypes. In contrast - to the ``psycopg`` multirange feature, SQLAlchemy's adaptation represents - a multirange datatype as a list of :class:`_postgresql.Range` objects. - -The example below illustrates use of the :class:`_postgresql.TSMULTIRANGE` -datatype:: - - from datetime import datetime - from typing import List - - from sqlalchemy.dialects.postgresql import Range - from sqlalchemy.dialects.postgresql import TSMULTIRANGE - from sqlalchemy.orm import DeclarativeBase - from sqlalchemy.orm import Mapped - from sqlalchemy.orm import mapped_column - - class Base(DeclarativeBase): - pass - - class EventCalendar(Base): - - __tablename__ = 'event_calendar' - - id: Mapped[int] = mapped_column(primary_key=True) - event_name: Mapped[str] - in_session_periods: Mapped[List[Range[datetime]]] = mapped_column(TSMULTIRANGE) - -Illustrating insertion and selecting of a record:: - - from sqlalchemy import create_engine - from sqlalchemy import select - from sqlalchemy.orm import Session - - engine = create_engine("postgresql+psycopg://scott:tiger@pg14/test") - - Base.metadata.create_all(engine) - - with Session(engine) as session: - calendar = EventCalendar( - event_name="SQLAlchemy Tutorial Sessions", - in_session_periods= [ - Range(datetime(2013, 3, 23), datetime(2013, 3, 25)), - Range(datetime(2013, 4, 12), datetime(2013, 4, 15)), - Range(datetime(2013, 5, 9), datetime(2013, 5, 12)), - ] - ) - session.add(calendar) - session.commit() - - for multirange in session.scalars(select(EventCalendar.in_session_periods)): - for range_ in multirange: - print(f"Start: {range_.lower} End: {range_.upper}") - -.. note:: In the above example, the list of :class:`_postgresql.Range` types - as handled by the ORM will not automatically detect in-place changes to - a particular list value; to update list values with the ORM, either re-assign - a new list to the attribute, or use the :class:`.MutableList` - type modifier. See the section :ref:`mutable_toplevel` for background. - - -The available multirange datatypes are as follows: - -* :class:`_postgresql.INT4MULTIRANGE` -* :class:`_postgresql.INT8MULTIRANGE` -* :class:`_postgresql.NUMMULTIRANGE` -* :class:`_postgresql.DATEMULTIRANGE` -* :class:`_postgresql.TSMULTIRANGE` -* :class:`_postgresql.TSTZMULTIRANGE` - - """ # noqa: E501