From 59521abcc0676e936b31a523bd968fc157fef0c2 Mon Sep 17 00:00:00 2001 From: Federico Caselli Date: Mon, 22 May 2023 21:54:47 +0200 Subject: [PATCH] add 3.12 Initial fixes to test to accommodate py312 Here we are pulling in a current fork of greenlet that works for python 3.12. It works though there is one issue with aiosqlite that might be related. a non-voting py312 job is added to gerrit / jenkins Fixes: #9819 Change-Id: I91a51dcbad2902f7c4c7cec88ebbf42c2417b512 --- .github/workflows/run-test.yaml | 6 ----- doc/build/changelog/unreleased_20/py312.rst | 4 +++ examples/versioned_history/history_meta.py | 2 +- lib/sqlalchemy/ext/asyncio/base.py | 2 +- lib/sqlalchemy/sql/sqltypes.py | 2 +- lib/sqlalchemy/util/__init__.py | 1 + lib/sqlalchemy/util/compat.py | 23 ++++++++++++++++ pyproject.toml | 6 ++++- test/base/test_result.py | 9 ++++++- test/engine/test_pool.py | 7 +++-- test/engine/test_reconnect.py | 30 ++++++++++----------- tox.ini | 21 ++++++++++----- 12 files changed, 78 insertions(+), 35 deletions(-) create mode 100644 doc/build/changelog/unreleased_20/py312.rst diff --git a/.github/workflows/run-test.yaml b/.github/workflows/run-test.yaml index 938c7c55dd..82818dc88b 100644 --- a/.github/workflows/run-test.yaml +++ b/.github/workflows/run-test.yaml @@ -51,12 +51,6 @@ jobs: pytest-args: "-k 'not test_autocommit_on and not test_turn_autocommit_off_via_default_iso_level and not test_autocommit_isolation_level'" - os: "ubuntu-latest" pytest-args: "--dbdriver pysqlite --dbdriver aiosqlite" - # do not run aiosqlite on python 3.11 - # wait for https://github.com/python/cpython/issues/99205 to be fixed - - os: "ubuntu-latest" - python-version: "3.11" - pytest-args: "--dbdriver pysqlite" - exclude: # linux and osx do not have x86 python diff --git a/doc/build/changelog/unreleased_20/py312.rst b/doc/build/changelog/unreleased_20/py312.rst new file mode 100644 index 0000000000..330cebb643 --- /dev/null +++ b/doc/build/changelog/unreleased_20/py312.rst @@ -0,0 +1,4 @@ +.. change:: + :tags: installation + + Compatibility improvements to work fully with Python 3.12 diff --git a/examples/versioned_history/history_meta.py b/examples/versioned_history/history_meta.py index c70a5bfa1c..806267cb41 100644 --- a/examples/versioned_history/history_meta.py +++ b/examples/versioned_history/history_meta.py @@ -113,7 +113,7 @@ def _history_mapper(local_mapper): Column( "changed", DateTime, - default=datetime.datetime.utcnow, + default=lambda: datetime.datetime.now(datetime.timezone.utc), info=version_meta, ) ) diff --git a/lib/sqlalchemy/ext/asyncio/base.py b/lib/sqlalchemy/ext/asyncio/base.py index 368c9b0553..1fecf60abd 100644 --- a/lib/sqlalchemy/ext/asyncio/base.py +++ b/lib/sqlalchemy/ext/asyncio/base.py @@ -184,7 +184,7 @@ class GeneratorStartableContext(StartableContext[_T_co]): # tell if we get the same exception back value = typ() try: - await self.gen.athrow(typ, value, traceback) + await util.athrow(self.gen, typ, value, traceback) except StopAsyncIteration as exc: # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration diff --git a/lib/sqlalchemy/sql/sqltypes.py b/lib/sqlalchemy/sql/sqltypes.py index f4ce48fcb2..5fdca7d0a7 100644 --- a/lib/sqlalchemy/sql/sqltypes.py +++ b/lib/sqlalchemy/sql/sqltypes.py @@ -2049,7 +2049,7 @@ class Interval(Emulated, _AbstractInterval, TypeDecorator[dt.timedelta]): """ impl = DateTime - epoch = dt.datetime.utcfromtimestamp(0) + epoch = dt.datetime.fromtimestamp(0, dt.timezone.utc).replace(tzinfo=None) cache_ok = True def __init__( diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 6f409c9e29..c804f96887 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -49,6 +49,7 @@ from ._collections import WeakPopulateDict as WeakPopulateDict from ._collections import WeakSequence as WeakSequence from .compat import anext_ as anext_ from .compat import arm as arm +from .compat import athrow as athrow from .compat import b as b from .compat import b64decode as b64decode from .compat import b64encode as b64encode diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index ab346083e5..da653b84d2 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -18,6 +18,8 @@ import platform import sys import typing from typing import Any +from typing import AsyncGenerator +from typing import Awaitable from typing import Callable from typing import Dict from typing import Iterable @@ -28,6 +30,7 @@ from typing import Sequence from typing import Set from typing import Tuple from typing import Type +from typing import TypeVar py312 = sys.version_info >= (3, 12) @@ -47,6 +50,8 @@ has_refcount_gc = bool(cpython) dottedgetter = operator.attrgetter +_T_co = TypeVar("_T_co", covariant=True) + class FullArgSpec(typing.NamedTuple): args: List[str] @@ -96,6 +101,24 @@ def inspect_getfullargspec(func: Callable[..., Any]) -> FullArgSpec: ) +if py312: + # we are 95% certain this form of athrow works in former Python + # versions, however we are unable to get confirmation; + # see https://github.com/python/cpython/issues/105269 where have + # been unable to get a straight answer so far + def athrow( # noqa + gen: AsyncGenerator[_T_co, Any], typ: Any, value: Any, traceback: Any + ) -> Awaitable[_T_co]: + return gen.athrow(value) + +else: + + def athrow( # noqa + gen: AsyncGenerator[_T_co, Any], typ: Any, value: Any, traceback: Any + ) -> Awaitable[_T_co]: + return gen.athrow(typ, value, traceback) + + if typing.TYPE_CHECKING or py38: from importlib import metadata as importlib_metadata else: diff --git a/pyproject.toml b/pyproject.toml index ae4c1fb136..a0c198c05d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,11 @@ filterwarnings = [ # here as pytest loads them immediately, which breaks coverage as well # as sys.path adjustments in conftest.py "error::DeprecationWarning:test", - "error::DeprecationWarning:sqlalchemy" + "error::DeprecationWarning:sqlalchemy", + + # sqlite3 warnings due to test/dialect/test_sqlite.py->test_native_datetime, + # which is asserting that these deprecated-in-py312 handlers are functional + "ignore:The default (date)?(time)?(stamp)? (adapter|converter):DeprecationWarning", ] markers = [ "memory_intensive: memory / CPU intensive suite tests", diff --git a/test/base/test_result.py b/test/base/test_result.py index b3f519bbef..78117e3228 100644 --- a/test/base/test_result.py +++ b/test/base/test_result.py @@ -7,7 +7,9 @@ from sqlalchemy.testing import eq_ from sqlalchemy.testing import fixtures from sqlalchemy.testing import is_false from sqlalchemy.testing import is_true +from sqlalchemy.testing.assertions import expect_raises from sqlalchemy.testing.util import picklers +from sqlalchemy.util import compat class ResultTupleTest(fixtures.TestBase): @@ -64,7 +66,12 @@ class ResultTupleTest(fixtures.TestBase): def test_slices_arent_in_mappings(self): keyed_tuple = self._fixture([1, 2], ["a", "b"]) - assert_raises(TypeError, lambda: keyed_tuple._mapping[0:2]) + if compat.py312: + with expect_raises(KeyError): + keyed_tuple._mapping[0:2] + else: + with expect_raises(TypeError): + keyed_tuple._mapping[0:2] def test_integers_arent_in_mappings(self): keyed_tuple = self._fixture([1, 2], ["a", "b"]) diff --git a/test/engine/test_pool.py b/test/engine/test_pool.py index cca6e25892..e2e4841740 100644 --- a/test/engine/test_pool.py +++ b/test/engine/test_pool.py @@ -736,7 +736,7 @@ class PoolEventsTest(PoolTestBase): assert canary.call_args_list[0][0][0] is dbapi_con assert canary.call_args_list[0][0][2] is exc - @testing.variation("is_asyncio", [True, False]) + @testing.variation("is_asyncio", [(True, testing.requires.asyncio), False]) @testing.variation("has_terminate", [True, False]) def test_checkin_event_gc(self, is_asyncio, has_terminate): """tests for #8419, which have been modified for 2.0 in #9237""" @@ -1704,7 +1704,10 @@ class QueuePoolTest(PoolTestBase): exc_cls=TimeoutThing if exc_type.base_exception else Exception, ) - @testing.combinations((True,), (False,)) + @testing.variation( + "detach_gced", + [("detached_gc", testing.requires.asyncio), "normal_gc"], + ) @testing.emits_warning("The garbage collector") def test_userspace_disconnectionerror_weakref_finalizer(self, detach_gced): dbapi, pool = self._queuepool_dbapi_fixture( diff --git a/test/engine/test_reconnect.py b/test/engine/test_reconnect.py index 48c5af8df7..00984b5a4d 100644 --- a/test/engine/test_reconnect.py +++ b/test/engine/test_reconnect.py @@ -1510,27 +1510,25 @@ class InvalidateDuringResultTest(fixtures.TestBase): self.meta.drop_all(conn) self.engine.dispose() - @testing.fails_if( - [ - "+mysqlconnector", - "+mysqldb", - "+cymysql", - "+pymysql", - "+pg8000", - "+asyncpg", - "+aiosqlite", - "+aiomysql", - "+asyncmy", - "+psycopg", - ], - "Buffers the result set and doesn't check for connection close", - ) def test_invalidate_on_results(self): conn = self.engine.connect() - result = conn.exec_driver_sql("select * from sometable") + result = conn.exec_driver_sql( + "select * from sometable", + ) for x in range(20): result.fetchone() + + real_cursor = result.cursor self.engine.test_shutdown() + + def produce_side_effect(): + # will fail because connection was closed, with an exception + # that should trigger disconnect routines + real_cursor.execute("select * from sometable") + + result.cursor = Mock( + fetchone=mock.Mock(side_effect=produce_side_effect) + ) try: _assert_invalidated(result.fetchone) assert conn.invalidated diff --git a/tox.ini b/tox.ini index d2d77353ab..31fc62aa1c 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ extras= sqlite_file: aiosqlite sqlite_file: sqlcipher; python_version < '3.10' postgresql: postgresql - postgresql: postgresql_asyncpg + py3{,7,8,9,10,11}-postgresql: postgresql_asyncpg postgresql: postgresql_pg8000 postgresql: postgresql_psycopg @@ -26,18 +26,19 @@ extras= mysql: pymysql mysql: asyncmy -# mysql: mariadb_connector - oracle: oracle oracle: oracle_oracledb - mssql: mssql - mssql: mssql_pymssql + py3{,7,8,9,10,11}-mssql: mssql + py3{,7,8,9,10,11}-mssql: mssql_pymssql + deps= pytest>=7.0.0rc1,<8 # tracked by https://github.com/pytest-dev/pytest-xdist/issues/907 pytest-xdist!=3.3.0 + py312: git+https://github.com/mdboom/greenlet.git@python-312#egg=greenlet + dbapimain-sqlite: git+https://github.com/omnilib/aiosqlite.git#egg=aiosqlite dbapimain-sqlite: git+https://github.com/coleifer/sqlcipher3.git#egg=sqlcipher3 @@ -82,10 +83,16 @@ setenv= PYTEST_COLOR={tty:--color=yes} MYPY_COLOR={tty:--color-output} - BASECOMMAND=python -m pytest {env:PYTEST_COLOR} --rootdir {toxinidir} --log-info=sqlalchemy.testing + # pytest 'rewrite' is hitting lots of deprecation warnings under py312 and + # i can't find any way to ignore those warnings, so this turns it off + py312: PYTEST_ARGS=--assert plain + + BASECOMMAND=python -m pytest {env:PYTEST_ARGS} {env:PYTEST_COLOR} --rootdir {toxinidir} --log-info=sqlalchemy.testing WORKERS={env:TOX_WORKERS:-n4 --max-worker-restart=5} + + nocext: DISABLE_SQLALCHEMY_CEXT=1 cext: REQUIRE_SQLALCHEMY_CEXT=1 cov: COVERAGE={[testenv]cov_args} @@ -107,7 +114,9 @@ setenv= py3{,10,11}-sqlite_file: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver aiosqlite} postgresql: POSTGRESQL={env:TOX_POSTGRESQL:--db postgresql} + py3{,7,8,9,10,11}-postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver asyncpg --dbdriver pg8000 --dbdriver psycopg --dbdriver psycopg_async} + py312-postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver pg8000 --dbdriver psycopg --dbdriver psycopg_async} mysql: MYSQL={env:TOX_MYSQL:--db mysql} py2{,7}-mysql: MYSQL={env:TOX_MYSQL_PY2K:{env:TOX_MYSQL:--db mysql}} -- 2.39.5