From: Nils Philippsen Date: Thu, 12 Jun 2025 13:32:48 +0000 (-0400) Subject: 1.4: Fix running tests with Python 3.14 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3bdfd0d3e493e704a7aca569a48fc743d934d169;p=thirdparty%2Fsqlalchemy%2Fsqlalchemy.git 1.4: Fix running tests with Python 3.14 Backported to SQLAlchemy 1.4 an improvement to the test suite with regards to how asyncio related tests are run, now using the newer Python 3.11 ``asyncio.Runner`` or a backported equivalent, rather than relying on the previous implementation based on ``asyncio.get_event_loop()``. This allows the SQLAlchemy 1.4 codebase to run on Python 3.14 which has removed this method. Pull request courtesy Nils Philippsen. Cleaned up some old cruft in tox.ini and added py314 markers. backported adef933f8d12938 from 2.0 to update test_memusage to use the "fork" context Fixes: #12668 Closes: #12666 Pull-request: https://github.com/sqlalchemy/sqlalchemy/pull/12666 Pull-request-sha: 6f0f4d6cca4cfcb60bd4ed09323eae5e5bd04958 Change-Id: I1f9682f945836e1e347cf1693eb30ff43ebe6e30 --- diff --git a/doc/build/changelog/unreleased_14/12668.rst b/doc/build/changelog/unreleased_14/12668.rst new file mode 100644 index 0000000000..40e2a1ff22 --- /dev/null +++ b/doc/build/changelog/unreleased_14/12668.rst @@ -0,0 +1,11 @@ +.. change:: + :tags: bug, tests + :tickets: 12668 + + Backported to SQLAlchemy 1.4 an improvement to the test suite with regards + to how asyncio related tests are run, now using the newer Python 3.11 + ``asyncio.Runner`` or a backported equivalent, rather than relying on the + previous implementation based on ``asyncio.get_event_loop()``. This allows + the SQLAlchemy 1.4 codebase to run on Python 3.14 which has removed this + method. Pull request courtesy Nils Philippsen. + diff --git a/lib/sqlalchemy/testing/asyncio.py b/lib/sqlalchemy/testing/asyncio.py index 5f15162002..6d4201c494 100644 --- a/lib/sqlalchemy/testing/asyncio.py +++ b/lib/sqlalchemy/testing/asyncio.py @@ -21,16 +21,21 @@ from functools import wraps import inspect from . import config -from ..util.concurrency import _util_async_run -from ..util.concurrency import _util_async_run_coroutine_function +from ..util.concurrency import _AsyncUtil # may be set to False if the # --disable-asyncio flag is passed to the test runner. ENABLE_ASYNCIO = True +_async_util = _AsyncUtil() # it has lazy init so just always create one + + +def _shutdown(): + """called when the test finishes""" + _async_util.close() def _run_coroutine_function(fn, *args, **kwargs): - return _util_async_run_coroutine_function(fn, *args, **kwargs) + return _async_util.run(fn, *args, **kwargs) def _assume_async(fn, *args, **kwargs): @@ -47,7 +52,7 @@ def _assume_async(fn, *args, **kwargs): if not ENABLE_ASYNCIO: return fn(*args, **kwargs) - return _util_async_run(fn, *args, **kwargs) + return _async_util.run_in_greenlet(fn, *args, **kwargs) def _maybe_async_provisioning(fn, *args, **kwargs): @@ -66,7 +71,7 @@ def _maybe_async_provisioning(fn, *args, **kwargs): return fn(*args, **kwargs) if config.any_async: - return _util_async_run(fn, *args, **kwargs) + return _async_util.run_in_greenlet(fn, *args, **kwargs) else: return fn(*args, **kwargs) @@ -87,7 +92,7 @@ def _maybe_async(fn, *args, **kwargs): is_async = config._current.is_async if is_async: - return _util_async_run(fn, *args, **kwargs) + return _async_util.run_in_greenlet(fn, *args, **kwargs) else: return fn(*args, **kwargs) diff --git a/lib/sqlalchemy/testing/plugin/pytestplugin.py b/lib/sqlalchemy/testing/plugin/pytestplugin.py index 2be6e6cda5..b33dcdb0d8 100644 --- a/lib/sqlalchemy/testing/plugin/pytestplugin.py +++ b/lib/sqlalchemy/testing/plugin/pytestplugin.py @@ -140,6 +140,12 @@ def pytest_sessionfinish(session): collect_types.dump_stats(session.config.option.dump_pyannotate) +def pytest_unconfigure(config): + from sqlalchemy.testing import asyncio + + asyncio._shutdown() + + def pytest_collection_finish(session): if session.config.option.dump_pyannotate: from pyannotate_runtime import collect_types diff --git a/lib/sqlalchemy/testing/requirements.py b/lib/sqlalchemy/testing/requirements.py index 9164faa93e..d8247dc283 100644 --- a/lib/sqlalchemy/testing/requirements.py +++ b/lib/sqlalchemy/testing/requirements.py @@ -1331,6 +1331,20 @@ class SuiteRequirements(Requirements): lambda: util.py38, "Python 3.8 or above required" ) + @property + def not_python314(self): + """This requirement is interim to assist with backporting of + issue #12405. + + SQLAlchemy 1.4 still includes the ``await_fallback()`` method that + makes use of ``asyncio.get_event_loop_policy()``. This is removed + in SQLAlchemy 2.1. + + """ + return exclusions.skip_if( + lambda: util.py314, "Python 3.14 or above not supported" + ) + @property def cpython(self): return exclusions.only_if( diff --git a/lib/sqlalchemy/util/__init__.py b/lib/sqlalchemy/util/__init__.py index 078723c048..b77f70f76a 100644 --- a/lib/sqlalchemy/util/__init__.py +++ b/lib/sqlalchemy/util/__init__.py @@ -82,6 +82,7 @@ from .compat import py310 from .compat import py311 from .compat import py312 from .compat import py313 +from .compat import py314 from .compat import py37 from .compat import py38 from .compat import py39 diff --git a/lib/sqlalchemy/util/_concurrency_py3k.py b/lib/sqlalchemy/util/_concurrency_py3k.py index 141193ef06..f20dcb05b5 100644 --- a/lib/sqlalchemy/util/_concurrency_py3k.py +++ b/lib/sqlalchemy/util/_concurrency_py3k.py @@ -10,12 +10,17 @@ import sys from typing import Any from typing import Callable from typing import Coroutine +from typing import TypeVar +from typing import Union import greenlet from . import compat from .langhelpers import memoized_property from .. import exc +from ..util import py311 + +_T = TypeVar("_T") # If greenlet.gr_context is present in current version of greenlet, # it will be set with the current context on creation. @@ -154,32 +159,6 @@ class AsyncAdaptedLock: self.mutex.release() -def _util_async_run_coroutine_function(fn, *args, **kwargs): - """for test suite/ util only""" - - loop = get_event_loop() - if loop.is_running(): - raise Exception( - "for async run coroutine we expect that no greenlet or event " - "loop is running when we start out" - ) - return loop.run_until_complete(fn(*args, **kwargs)) - - -def _util_async_run(fn, *args, **kwargs): - """for test suite/ util only""" - - loop = get_event_loop() - if not loop.is_running(): - return loop.run_until_complete(greenlet_spawn(fn, *args, **kwargs)) - else: - # allow for a wrapped test function to call another - assert getattr( - greenlet.getcurrent(), "__sqlalchemy_greenlet_provider__", False - ) - return fn(*args, **kwargs) - - def get_event_loop(): """vendor asyncio.get_event_loop() for python 3.7 and above. @@ -193,3 +172,50 @@ def get_event_loop(): return asyncio.get_event_loop_policy().get_event_loop() else: return asyncio.get_event_loop() + + +if py311: + _Runner = asyncio.Runner +else: + + class _Runner: # type: ignore[no-redef] + """Runner implementation for test only""" + + _loop: Union[None, asyncio.AbstractEventLoop, bool] + + def __init__(self) -> None: + self._loop = None + + def __enter__(self): + self._lazy_init() + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.close() + + def close(self) -> None: + if self._loop: + try: + self._loop.run_until_complete( + self._loop.shutdown_asyncgens() + ) + finally: + self._loop.close() + self._loop = False + + def get_loop(self) -> asyncio.AbstractEventLoop: + """Return embedded event loop.""" + self._lazy_init() + assert self._loop + return self._loop + + def run(self, coro: Coroutine[Any, Any, _T]) -> _T: + self._lazy_init() + assert self._loop + return self._loop.run_until_complete(coro) + + def _lazy_init(self) -> None: + if self._loop is False: + raise RuntimeError("Runner is closed") + if self._loop is None: + self._loop = asyncio.new_event_loop() diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index bdb4a97854..fe7e7d6352 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -14,6 +14,7 @@ import operator import platform import sys +py314 = sys.version_info >= (3, 14) py313 = sys.version_info >= (3, 13) py312 = sys.version_info >= (3, 12) py311 = sys.version_info >= (3, 11) diff --git a/lib/sqlalchemy/util/concurrency.py b/lib/sqlalchemy/util/concurrency.py index 7341dbe685..8258085216 100644 --- a/lib/sqlalchemy/util/concurrency.py +++ b/lib/sqlalchemy/util/concurrency.py @@ -22,15 +22,39 @@ if compat.py3k: from ._concurrency_py3k import greenlet_spawn from ._concurrency_py3k import is_exit_exception from ._concurrency_py3k import AsyncAdaptedLock - from ._concurrency_py3k import _util_async_run # noqa: F401 - from ._concurrency_py3k import ( - _util_async_run_coroutine_function, - ) # noqa: F401, E501 + from ._concurrency_py3k import _Runner from ._concurrency_py3k import asyncio # noqa: F401 - # does not need greennlet, just Python 3 + # does not need greenlet, just Python 3 from ._compat_py3k import asynccontextmanager # noqa: F401 + +class _AsyncUtil: + """Asyncio util for test suite/ util only""" + + def __init__(self): + if have_greenlet: + self.runner = _Runner() + + def run(self, fn, *args, **kwargs): + """Run coroutine on the loop""" + return self.runner.run(fn(*args, **kwargs)) + + def run_in_greenlet(self, fn, *args, **kwargs): + """Run sync function in greenlet. Support nested calls""" + if have_greenlet: + if self.runner.get_loop().is_running(): + return fn(*args, **kwargs) + else: + return self.runner.run(greenlet_spawn(fn, *args, **kwargs)) + else: + return fn(*args, **kwargs) + + def close(self): + if have_greenlet: + self.runner.close() + + if not have_greenlet: asyncio = None # noqa: F811 diff --git a/setup.cfg b/setup.cfg index b4dd728ead..e0752d1feb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,8 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + Programming Language :: Python :: 3.14 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: Database :: Front-Ends @@ -139,7 +141,7 @@ ignore = A003, A004, A005, A006 D, E203,E305,E711,E712,E721,E722,E741, - FA100, + FA100,F824 N801,N802,N806, RST304,RST303,RST299,RST399, W503,W504 diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py index 4b2699e1be..bcf3475e9d 100644 --- a/test/aaa_profiling/test_memusage.py +++ b/test/aaa_profiling/test_memusage.py @@ -211,10 +211,14 @@ def profile_memory( # return run_plain def run_in_process(*func_args): - queue = multiprocessing.Queue() - proc = multiprocessing.Process( - target=profile, args=(queue, func_args) - ) + # see + # https://docs.python.org/3.14/whatsnew/3.14.html + # #incompatible-changes - the default run type is no longer + # "fork", but since we are running closures in the process + # we need forked mode + ctx = multiprocessing.get_context("fork") + queue = ctx.Queue() + proc = ctx.Process(target=profile, args=(queue, func_args)) proc.start() while True: row = queue.get() diff --git a/test/base/test_concurrency_py3k.py b/test/base/test_concurrency_py3k.py index de7157c788..c639f24feb 100644 --- a/test/base/test_concurrency_py3k.py +++ b/test/base/test_concurrency_py3k.py @@ -81,6 +81,7 @@ class TestAsyncioCompat(fixtures.TestBase): with expect_raises_message(ValueError, "sync error"): await greenlet_spawn(go) + @testing.requires.not_python314 def test_await_fallback_no_greenlet(self): to_await = run1() await_fallback(to_await) diff --git a/tox.ini b/tox.ini index 3adbe51dbd..071415c4f9 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ usedevelop= cov: True extras= - py{3,38,39,310,311,312}: {[greenletextras]extras} + py{3,38,39,310,311,312,313,314}: {[greenletextras]extras} postgresql: postgresql postgresql: postgresql-pg8000 @@ -50,7 +50,6 @@ deps= py313: git+https://github.com/python-greenlet/greenlet.git\#egg=greenlet dbapimain-sqlite: git+https://github.com/omnilib/aiosqlite.git\#egg=aiosqlite - dbapimain-sqlite: git+https://github.com/coleifer/sqlcipher3.git\#egg=sqlcipher3 dbapimain-postgresql: git+https://github.com/psycopg/psycopg2.git#egg=psycopg2 dbapimain-postgresql: git+https://github.com/MagicStack/asyncpg.git#egg=asyncpg @@ -58,11 +57,9 @@ deps= dbapimain-mysql: git+https://github.com/PyMySQL/mysqlclient-python.git#egg=mysqlclient dbapimain-mysql: git+https://github.com/PyMySQL/PyMySQL.git#egg=pymysql - # dbapimain-mysql: git+https://github.com/mariadb-corporation/mariadb-connector-python#egg=mariadb dbapimain-oracle: git+https://github.com/oracle/python-cx_Oracle.git#egg=cx_Oracle - py313-mssql: git+https://github.com/mkleehammer/pyodbc.git\#egg=pyodbc dbapimain-mssql: git+https://github.com/mkleehammer/pyodbc.git\#egg=pyodbc cov: pytest-cov @@ -89,10 +86,6 @@ setenv= PYTEST_COLOR={tty:--color=yes} - # pytest 'rewrite' is hitting lots of deprecation warnings under py312 and - # i can't find any way to ignore those warnings, so this turns it off - py312: PYTEST_ARGS=--assert plain - MEMUSAGE=--nomemory BASECOMMAND=python -m pytest {env:PYTEST_COLOR} --rootdir {toxinidir} --log-info=sqlalchemy.testing @@ -115,14 +108,11 @@ setenv= sqlite-nogreenlet: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver pysqlite_numeric} - py{37,38,39}-sqlite_file: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver aiosqlite --dbdriver pysqlcipher} - - # omit pysqlcipher for Python 3.10 - py3{,10,11}-sqlite_file: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver aiosqlite} + py-sqlite_file: EXTRA_SQLITE_DRIVERS={env:EXTRA_SQLITE_DRIVERS:--dbdriver sqlite --dbdriver aiosqlite} postgresql: POSTGRESQL={env:TOX_POSTGRESQL:--db postgresql} py2{,7}-postgresql: POSTGRESQL={env:TOX_POSTGRESQL_PY2K:{env:TOX_POSTGRESQL:--db postgresql}} - py3{,5,6,7,8,9,10,11}-postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver asyncpg --dbdriver pg8000} + py3{,5,6,7,8,9,10,11,12,13,14}-postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver asyncpg --dbdriver pg8000} py312-postgresql: EXTRA_PG_DRIVERS={env:EXTRA_PG_DRIVERS:--dbdriver psycopg2 --dbdriver pg8000} mysql: MYSQL={env:TOX_MYSQL:--db mysql}