From 150ebad66da7767416da7cae4871e7532abc352f Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Tue, 28 Nov 2023 12:46:59 +0000 Subject: [PATCH] chore: add support for PyPy. --- .github/workflows/tests.yml | 9 ++++ docs/basic/install.rst | 14 ++++- docs/news.rst | 1 + psycopg/psycopg/pq/pq_ctypes.py | 5 +- psycopg/setup.cfg | 2 + psycopg/setup.py | 4 +- psycopg_c/setup.cfg | 2 + psycopg_pool/setup.cfg | 2 + tests/conftest.py | 70 ++++++++++++++++++++++++- tests/crdb/test_copy.py | 10 ++-- tests/crdb/test_copy_async.py | 10 ++-- tests/fix_pq.py | 2 +- tests/pool/test_null_pool.py | 3 +- tests/pool/test_pool.py | 6 ++- tests/pool/test_pool_async_noasyncio.py | 4 +- tests/pq/test_pgconn.py | 4 +- tests/test_client_cursor.py | 11 ++-- tests/test_client_cursor_async.py | 12 ++--- tests/test_connection.py | 9 ++-- tests/test_connection_async.py | 10 ++-- tests/test_copy.py | 18 +++---- tests/test_copy_async.py | 18 +++---- tests/test_cursor.py | 12 ++--- tests/test_cursor_async.py | 11 ++-- tests/test_errors.py | 4 +- tests/test_server_cursor.py | 3 +- tests/test_server_cursor_async.py | 7 ++- tests/types/test_array.py | 4 +- tests/utils.py | 42 --------------- 29 files changed, 184 insertions(+), 125 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ac293782f..93a2138f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,6 +48,10 @@ jobs: # Test with minimum dependencies versions - {impl: c, python: "3.7", ext: min, postgres: "postgres:15"} + # Test with PyPy. + - {impl: python, python: "pypy3.9", postgres: "postgres:13"} + - {impl: python, python: "pypy3.10", postgres: "postgres:14"} + env: PSYCOPG_IMPL: ${{ matrix.impl }} DEPS: ./psycopg[test] ./psycopg_pool @@ -96,6 +100,11 @@ jobs: echo "DEPS=$DEPS shapely" >> $GITHUB_ENV echo "MARKERS=$MARKERS postgis" >> $GITHUB_ENV + - name: Exclude certain tests from pypy + if: ${{ startsWith(matrix.python, 'pypy') }} + run: | + echo "NOT_MARKERS=$NOT_MARKERS timing" >> $GITHUB_ENV + - name: Configure to use the oldest dependencies if: ${{ matrix.ext == 'min' }} run: | diff --git a/docs/basic/install.rst b/docs/basic/install.rst index 120c08f4a..7881f3b66 100644 --- a/docs/basic/install.rst +++ b/docs/basic/install.rst @@ -6,7 +6,7 @@ Installation In short, if you use a :ref:`supported system`:: pip install --upgrade pip # upgrade pip to at least 20.3 - pip install "psycopg[binary]" + pip install "psycopg[binary]" # remove [binary] for PyPy and you should be :ref:`ready to start `. Read further for alternative ways to install. @@ -23,6 +23,10 @@ The Psycopg version documented here has *official and tested* support for: - Python 3.6 supported before Psycopg 3.1 +- PyPy: from version 3.9 to 3.10 + + - **Note:** Only the pure Python version is supported. + - PostgreSQL: from version 10 to 16 - OS: Linux, macOS, Windows @@ -72,6 +76,10 @@ installation ` or a :ref:`pure Python installation For further information about the differences between the packages see :ref:`pq-impl`. +.. warning:: + + The binary installation is not supported by PyPy. + .. _local-installation: @@ -99,6 +107,10 @@ If your build prerequisites are in place you can run:: pip install "psycopg[c]" +.. warning:: + + The local installation is not supported by PyPy. + .. _pure-python-installation: diff --git a/docs/news.rst b/docs/news.rst index 814c244bc..5344e3269 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -14,6 +14,7 @@ Psycopg 3.1.14 (unreleased) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Fix :ref:`interaction with gevent ` (:ticket:`#527`). +- Add support for PyPy (:ticket:`#686`). .. _gevent: https://www.gevent.org/ diff --git a/psycopg/psycopg/pq/pq_ctypes.py b/psycopg/psycopg/pq/pq_ctypes.py index fdf2ad8a8..96c586174 100644 --- a/psycopg/psycopg/pq/pq_ctypes.py +++ b/psycopg/psycopg/pq/pq_ctypes.py @@ -47,7 +47,10 @@ def version() -> int: @impl.PQnoticeReceiver # type: ignore def notice_receiver(arg: c_void_p, result_ptr: impl.PGresult_struct) -> None: - pgconn = cast(arg, POINTER(py_object)).contents.value() + pgconn = cast(arg, POINTER(py_object)).contents.value + if callable(pgconn): # Not a weak reference on PyPy. + pgconn = pgconn() + if not (pgconn and pgconn.notice_handler): return diff --git a/psycopg/setup.cfg b/psycopg/setup.cfg index 536447e43..ece28036f 100644 --- a/psycopg/setup.cfg +++ b/psycopg/setup.cfg @@ -28,6 +28,8 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy Topic :: Database Topic :: Database :: Front-Ends Topic :: Software Development diff --git a/psycopg/setup.py b/psycopg/setup.py index c699c872b..5cc6d1a8e 100644 --- a/psycopg/setup.py +++ b/psycopg/setup.py @@ -25,11 +25,11 @@ with open("psycopg/version.py") as f: extras_require = { # Install the C extension module (requires dev tools) "c": [ - f"psycopg-c == {version}", + f"psycopg-c == {version}; implementation_name != 'pypy'", ], # Install the stand-alone C extension module "binary": [ - f"psycopg-binary == {version}", + f"psycopg-binary == {version}; implementation_name != 'pypy'", ], # Install the connection pool "pool": [ diff --git a/psycopg_c/setup.cfg b/psycopg_c/setup.cfg index e6875dcd8..ccd8936cf 100644 --- a/psycopg_c/setup.cfg +++ b/psycopg_c/setup.cfg @@ -21,6 +21,7 @@ classifiers = Operating System :: MacOS :: MacOS X Operating System :: Microsoft :: Windows Operating System :: POSIX + Programming Language :: Cython Programming Language :: Python :: 3 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 @@ -28,6 +29,7 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: Implementation :: CPython Topic :: Database Topic :: Database :: Front-Ends Topic :: Software Development diff --git a/psycopg_pool/setup.cfg b/psycopg_pool/setup.cfg index 48380bdb5..bc04073d6 100644 --- a/psycopg_pool/setup.cfg +++ b/psycopg_pool/setup.cfg @@ -28,6 +28,8 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy Topic :: Database Topic :: Database :: Front-Ends Topic :: Software Development diff --git a/tests/conftest.py b/tests/conftest.py index de2b24fc7..f5d162357 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ +import gc import sys import asyncio import selectors -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple import pytest @@ -97,3 +98,70 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): terminalreporter.section("failed tests ignored") for msg in allow_fail_messages: terminalreporter.line(msg) + + +NO_COUNT_TYPES: Tuple[type, ...] = () + +if sys.version_info[:2] == (3, 10): + # On my laptop there are occasional creations of a single one of these objects + # with empty content, which might be some Decimal caching. + # Keeping the guard as strict as possible, to be extended if other types + # or versions are necessary. + try: + from _contextvars import Context # type: ignore + except ImportError: + pass + else: + NO_COUNT_TYPES += (Context,) + + +class GCFixture: + __slots__ = () + + @staticmethod + def collect() -> None: + """ + gc.collect(), but more insisting. + """ + for i in range(3): + gc.collect() + + @staticmethod + def count() -> int: + """ + len(gc.get_objects()), with subtleties. + """ + + if not NO_COUNT_TYPES: + return len(gc.get_objects()) + + # Note: not using a list comprehension because it pollutes the objects list. + rv = 0 + for obj in gc.get_objects(): + if isinstance(obj, NO_COUNT_TYPES): + continue + rv += 1 + + return rv + + +@pytest.fixture(name="gc") +def fixture_gc(): + """ + Provides a consistent way to run garbage collection and count references. + + **Note:** This will skip tests on PyPy. + """ + if sys.implementation.name == "pypy": + pytest.skip(reason="depends on refcount semantics") + return GCFixture() + + +@pytest.fixture +def gc_collect(): + """ + Provides a consistent way to run garbage collection. + + **Note:** This will *not* skip tests on PyPy. + """ + return GCFixture.collect diff --git a/tests/crdb/test_copy.py b/tests/crdb/test_copy.py index b7d26aa51..c3ccdfe11 100644 --- a/tests/crdb/test_copy.py +++ b/tests/crdb/test_copy.py @@ -7,7 +7,7 @@ from psycopg.pq import Format from psycopg.adapt import PyFormat from psycopg.types.numeric import Int4 -from ..utils import eur, gc_collect, gc_count +from ..utils import eur from ..test_copy import sample_text, sample_binary # noqa from ..test_copy import ensure_table, sample_records from ..test_copy import sample_tabledef as sample_tabledef_pg @@ -191,7 +191,7 @@ from copy_in group by 1, 2, 3 [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) @pytest.mark.crdb_skip("copy array") -def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): +def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -219,12 +219,12 @@ def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): for got, want in zip(recs, faker.records): faker.assert_record(got, want) - gc_collect() + gc.collect() n = [] for i in range(3): work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/crdb/test_copy_async.py b/tests/crdb/test_copy_async.py index 45ee5eca0..06a702664 100644 --- a/tests/crdb/test_copy_async.py +++ b/tests/crdb/test_copy_async.py @@ -7,7 +7,7 @@ from psycopg import sql, errors as e from psycopg.adapt import PyFormat from psycopg.types.numeric import Int4 -from ..utils import eur, gc_collect, gc_count +from ..utils import eur from ..test_copy import sample_text, sample_binary # noqa from ..test_copy import sample_records from ..test_copy_async import ensure_table @@ -197,7 +197,7 @@ from copy_in group by 1, 2, 3 [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) @pytest.mark.crdb_skip("copy array") -async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): +async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -225,11 +225,11 @@ async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): for got, want in zip(recs, faker.records): faker.assert_record(got, want) - gc_collect() + gc.collect() n = [] for i in range(3): await work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/fix_pq.py b/tests/fix_pq.py index 6811a26c3..917dfc919 100644 --- a/tests/fix_pq.py +++ b/tests/fix_pq.py @@ -53,7 +53,7 @@ def libpq(): # Not available when testing the binary package libname = find_libpq_full_path() assert libname, "libpq libname not found" - return ctypes.pydll.LoadLibrary(libname) + return ctypes.cdll.LoadLibrary(libname) except Exception as e: if pq.__impl__ == "binary": pytest.skip(f"can't load libpq for testing: {e}") diff --git a/tests/pool/test_null_pool.py b/tests/pool/test_null_pool.py index 51cc67ad9..68a03ddf7 100644 --- a/tests/pool/test_null_pool.py +++ b/tests/pool/test_null_pool.py @@ -618,12 +618,13 @@ def test_putconn_wrong_pool(dsn): @pytest.mark.slow -def test_del_stop_threads(dsn): +def test_del_stop_threads(dsn, gc): p = NullConnectionPool(dsn) assert p._sched_runner is not None ts = [p._sched_runner] + p._workers del p sleep(0.1) + gc.collect() for t in ts: assert not t.is_alive() diff --git a/tests/pool/test_pool.py b/tests/pool/test_pool.py index 5b4c435d7..8eb891144 100644 --- a/tests/pool/test_pool.py +++ b/tests/pool/test_pool.py @@ -588,7 +588,7 @@ def test_putconn_wrong_pool(dsn): p2.putconn(conn) -def test_del_no_warning(dsn, recwarn): +def test_del_no_warning(dsn, recwarn, gc_collect): p = pool.ConnectionPool(dsn, min_size=2) with p.connection() as conn: conn.execute("select 1") @@ -596,17 +596,19 @@ def test_del_no_warning(dsn, recwarn): p.wait() ref = weakref.ref(p) del p + gc_collect() assert not ref() assert not recwarn, [str(w.message) for w in recwarn.list] @pytest.mark.slow -def test_del_stop_threads(dsn): +def test_del_stop_threads(dsn, gc): p = pool.ConnectionPool(dsn) assert p._sched_runner is not None ts = [p._sched_runner] + p._workers del p sleep(0.1) + gc.collect() for t in ts: assert not t.is_alive() diff --git a/tests/pool/test_pool_async_noasyncio.py b/tests/pool/test_pool_async_noasyncio.py index 982b9288e..c820209c6 100644 --- a/tests/pool/test_pool_async_noasyncio.py +++ b/tests/pool/test_pool_async_noasyncio.py @@ -5,8 +5,6 @@ import asyncio import pytest -from ..utils import gc_collect - try: import psycopg_pool as pool except ImportError: @@ -61,7 +59,7 @@ def test_cant_create_open_outside_loop(dsn): @pytest.fixture -def asyncio_run(anyio_backend_options, recwarn): +def asyncio_run(anyio_backend_options, recwarn, gc_collect): """Fixture reuturning asyncio.run, but managing resources at exit. In certain runs, fd objects are leaked and the error will only be caught diff --git a/tests/pq/test_pgconn.py b/tests/pq/test_pgconn.py index 05661511a..ff18379a4 100644 --- a/tests/pq/test_pgconn.py +++ b/tests/pq/test_pgconn.py @@ -11,8 +11,6 @@ import psycopg from psycopg import pq import psycopg.generators -from ..utils import gc_collect - def test_connectdb(dsn): conn = pq.PGconn.connect(dsn.encode()) @@ -82,7 +80,7 @@ def test_finish(pgconn): @pytest.mark.slow -def test_weakref(dsn): +def test_weakref(dsn, gc_collect): conn = pq.PGconn.connect(dsn.encode()) w = weakref.ref(conn) conn.finish() diff --git a/tests/test_client_cursor.py b/tests/test_client_cursor.py index 3d6ec4dbf..416f9fbde 100644 --- a/tests/test_client_cursor.py +++ b/tests/test_client_cursor.py @@ -11,7 +11,6 @@ from psycopg.adapt import PyFormat from psycopg.postgres import types as builtins from psycopg.types import TypeInfo -from .utils import gc_collect, gc_count from .test_cursor import my_row_factory from .fix_crdb import is_crdb, crdb_encoding, crdb_time_precision @@ -117,7 +116,7 @@ def test_context(conn): @pytest.mark.slow -def test_weakref(conn): +def test_weakref(conn, gc_collect): cur = conn.cursor() w = weakref.ref(cur) cur.close() @@ -793,7 +792,7 @@ def test_str(conn): @pytest.mark.slow @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) @pytest.mark.parametrize("row_factory", ["tuple_row", "dict_row", "namedtuple_row"]) -def test_leak(conn_cls, dsn, faker, fetch, row_factory): +def test_leak(conn_cls, dsn, faker, fetch, row_factory, gc): faker.choose_schema(ncols=5) faker.make_records(10) row_factory = getattr(rows, row_factory) @@ -825,11 +824,11 @@ def test_leak(conn_cls, dsn, faker, fetch, row_factory): pass n = [] - gc_collect() + gc.collect() for i in range(3): work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_client_cursor_async.py b/tests/test_client_cursor_async.py index 67b314acb..e7b55d4a4 100644 --- a/tests/test_client_cursor_async.py +++ b/tests/test_client_cursor_async.py @@ -8,7 +8,7 @@ from psycopg import sql, rows from psycopg.adapt import PyFormat from psycopg.types import TypeInfo -from .utils import alist, gc_collect, gc_count +from .utils import alist from .test_cursor import my_row_factory from .test_cursor import execmany, _execmany # noqa: F401 from .fix_crdb import crdb_encoding @@ -120,7 +120,7 @@ async def test_context(aconn): @pytest.mark.slow -async def test_weakref(aconn): +async def test_weakref(aconn, gc_collect): cur = aconn.cursor() w = weakref.ref(cur) await cur.close() @@ -638,7 +638,7 @@ async def test_str(aconn): @pytest.mark.slow @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) @pytest.mark.parametrize("row_factory", ["tuple_row", "dict_row", "namedtuple_row"]) -async def test_leak(aconn_cls, dsn, faker, fetch, row_factory): +async def test_leak(aconn_cls, dsn, faker, fetch, row_factory, gc): faker.choose_schema(ncols=5) faker.make_records(10) row_factory = getattr(rows, row_factory) @@ -671,11 +671,11 @@ async def test_leak(aconn_cls, dsn, faker, fetch, row_factory): pass n = [] - gc_collect() + gc.collect() for i in range(3): await work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_connection.py b/tests/test_connection.py index 10f0a6ca7..3df8475d8 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -11,7 +11,6 @@ from psycopg import Notify, pq, errors as e from psycopg.rows import tuple_row from psycopg.conninfo import conninfo_to_dict, make_conninfo -from .utils import gc_collect from .test_cursor import my_row_factory from .test_adapt import make_bin_dumper, make_dumper @@ -127,19 +126,22 @@ def test_cursor_closed(conn): (pq.__impl__ in ("c", "binary") and sys.version_info[:2] == (3, 12)), reason="Something with Exceptions, C, Python 3.12", ) -def test_connection_warn_close(conn_cls, dsn, recwarn): +def test_connection_warn_close(conn_cls, dsn, recwarn, gc_collect): conn = conn_cls.connect(dsn) conn.close() del conn + gc_collect() assert not recwarn, [str(w.message) for w in recwarn.list] conn = conn_cls.connect(dsn) del conn + gc_collect() assert "IDLE" in str(recwarn.pop(ResourceWarning).message) conn = conn_cls.connect(dsn) conn.execute("select 1") del conn + gc_collect() assert "INTRANS" in str(recwarn.pop(ResourceWarning).message) conn = conn_cls.connect(dsn) @@ -154,6 +156,7 @@ def test_connection_warn_close(conn_cls, dsn, recwarn): with conn_cls.connect(dsn) as conn: pass del conn + gc_collect() assert not recwarn, [str(w.message) for w in recwarn.list] @@ -238,7 +241,7 @@ def test_context_active_rollback_no_clobber(conn_cls, dsn, caplog): @pytest.mark.slow -def test_weakref(conn_cls, dsn): +def test_weakref(conn_cls, dsn, gc_collect): conn = conn_cls.connect(dsn) w = weakref.ref(conn) conn.close() diff --git a/tests/test_connection_async.py b/tests/test_connection_async.py index b4c100b9b..17ecbe3d6 100644 --- a/tests/test_connection_async.py +++ b/tests/test_connection_async.py @@ -9,7 +9,6 @@ from psycopg import Notify, errors as e from psycopg.rows import tuple_row from psycopg.conninfo import conninfo_to_dict, make_conninfo -from .utils import gc_collect from .test_cursor import my_row_factory from .test_connection import tx_params, tx_params_isolation, tx_values_map from .test_connection import conninfo_params_timeout, drop_default_args_from_conninfo @@ -126,19 +125,22 @@ async def test_cursor_closed(aconn): aconn.cursor() -async def test_connection_warn_close(aconn_cls, dsn, recwarn): +async def test_connection_warn_close(aconn_cls, dsn, recwarn, gc_collect): conn = await aconn_cls.connect(dsn) await conn.close() del conn + gc_collect() assert not recwarn, [str(w.message) for w in recwarn.list] conn = await aconn_cls.connect(dsn) del conn + gc_collect() assert "IDLE" in str(recwarn.pop(ResourceWarning).message) conn = await aconn_cls.connect(dsn) await conn.execute("select 1") del conn + gc_collect() assert "INTRANS" in str(recwarn.pop(ResourceWarning).message) conn = await aconn_cls.connect(dsn) @@ -147,11 +149,13 @@ async def test_connection_warn_close(aconn_cls, dsn, recwarn): except Exception: pass del conn + gc_collect() assert "INERROR" in str(recwarn.pop(ResourceWarning).message) async with await aconn_cls.connect(dsn) as conn: pass del conn + gc_collect() assert not recwarn, [str(w.message) for w in recwarn.list] @@ -229,7 +233,7 @@ async def test_context_active_rollback_no_clobber(aconn_cls, dsn, caplog): @pytest.mark.slow -async def test_weakref(aconn_cls, dsn): +async def test_weakref(aconn_cls, dsn, gc_collect): conn = await aconn_cls.connect(dsn) w = weakref.ref(conn) await conn.close() diff --git a/tests/test_copy.py b/tests/test_copy.py index 2c21368ae..bc3628c6a 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -18,7 +18,7 @@ from psycopg.types import TypeInfo from psycopg.types.hstore import register_hstore from psycopg.types.numeric import Int4 -from .utils import eur, gc_collect, gc_count +from .utils import eur pytestmark = pytest.mark.crdb_skip("copy") @@ -691,7 +691,7 @@ def test_connection_writer(conn, format, buffer): [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) @pytest.mark.parametrize("method", ["read", "iter", "row", "rows"]) -def test_copy_to_leaks(conn_cls, dsn, faker, fmt, set_types, method): +def test_copy_to_leaks(conn_cls, dsn, faker, fmt, set_types, method, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -731,12 +731,12 @@ def test_copy_to_leaks(conn_cls, dsn, faker, fmt, set_types, method): elif method == "rows": list(copy.rows()) - gc_collect() + gc.collect() n = [] for i in range(3): work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" @@ -746,7 +746,7 @@ def test_copy_to_leaks(conn_cls, dsn, faker, fmt, set_types, method): "fmt, set_types", [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) -def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): +def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -774,12 +774,12 @@ def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): for got, want in zip(recs, faker.records): faker.assert_record(got, want) - gc_collect() + gc.collect() n = [] for i in range(3): work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_copy_async.py b/tests/test_copy_async.py index dd11d4bd2..ea8f23028 100644 --- a/tests/test_copy_async.py +++ b/tests/test_copy_async.py @@ -18,7 +18,7 @@ from psycopg.adapt import PyFormat from psycopg.types.hstore import register_hstore from psycopg.types.numeric import Int4 -from .utils import alist, eur, gc_collect, gc_count +from .utils import alist, eur from .test_copy import sample_text, sample_binary, sample_binary_rows # noqa from .test_copy import sample_values, sample_records, sample_tabledef from .test_copy import py_to_raw, special_chars @@ -697,7 +697,7 @@ async def test_connection_writer(aconn, format, buffer): [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) @pytest.mark.parametrize("method", ["read", "iter", "row", "rows"]) -async def test_copy_to_leaks(aconn_cls, dsn, faker, fmt, set_types, method): +async def test_copy_to_leaks(aconn_cls, dsn, faker, fmt, set_types, method, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -737,12 +737,12 @@ async def test_copy_to_leaks(aconn_cls, dsn, faker, fmt, set_types, method): elif method == "rows": await alist(copy.rows()) - gc_collect() + gc.collect() n = [] for i in range(3): await work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" @@ -752,7 +752,7 @@ async def test_copy_to_leaks(aconn_cls, dsn, faker, fmt, set_types, method): "fmt, set_types", [(Format.TEXT, True), (Format.TEXT, False), (Format.BINARY, True)], ) -async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): +async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types, gc): faker.format = PyFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -780,12 +780,12 @@ async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): for got, want in zip(recs, faker.records): faker.assert_record(got, want) - gc_collect() + gc.collect() n = [] for i in range(3): await work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_cursor.py b/tests/test_cursor.py index f48714715..c217acf82 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -12,7 +12,6 @@ from psycopg.adapt import PyFormat from psycopg.postgres import types as builtins from psycopg.rows import RowMaker -from .utils import gc_collect, gc_count from .fix_crdb import is_crdb, crdb_encoding, crdb_time_precision @@ -100,7 +99,7 @@ def test_context(conn): @pytest.mark.slow -def test_weakref(conn): +def test_weakref(conn, gc_collect): cur = conn.cursor() w = weakref.ref(cur) cur.close() @@ -913,7 +912,7 @@ def test_str(conn): @pytest.mark.parametrize("fmt_out", pq.Format) @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) @pytest.mark.parametrize("row_factory", ["tuple_row", "dict_row", "namedtuple_row"]) -def test_leak(conn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): +def test_leak(conn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory, gc): faker.format = fmt faker.choose_schema(ncols=5) faker.make_records(10) @@ -946,11 +945,12 @@ def test_leak(conn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): pass n = [] - gc_collect() + gc.collect() for i in range(3): work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) + assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_cursor_async.py b/tests/test_cursor_async.py index e20815355..6427064ef 100644 --- a/tests/test_cursor_async.py +++ b/tests/test_cursor_async.py @@ -7,7 +7,6 @@ import psycopg from psycopg import pq, sql, rows from psycopg.adapt import PyFormat -from .utils import gc_collect, gc_count from .test_cursor import my_row_factory from .test_cursor import execmany, _execmany # noqa: F401 from .fix_crdb import crdb_encoding @@ -99,7 +98,7 @@ async def test_context(aconn): @pytest.mark.slow -async def test_weakref(aconn): +async def test_weakref(aconn, gc_collect): cur = aconn.cursor() w = weakref.ref(cur) await cur.close() @@ -760,7 +759,7 @@ async def test_str(aconn): @pytest.mark.parametrize("fmt_out", pq.Format) @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) @pytest.mark.parametrize("row_factory", ["tuple_row", "dict_row", "namedtuple_row"]) -async def test_leak(aconn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): +async def test_leak(aconn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory, gc): faker.format = fmt faker.choose_schema(ncols=5) faker.make_records(10) @@ -794,10 +793,10 @@ async def test_leak(aconn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): pass n = [] - gc_collect() + gc.collect() for i in range(3): await work() - gc_collect() - n.append(gc_count()) + gc.collect() + n.append(gc.count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_errors.py b/tests/test_errors.py index ddf57513c..a5016ae32 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -9,7 +9,7 @@ import psycopg from psycopg import pq from psycopg import errors as e -from .utils import eur, gc_collect +from .utils import eur from .fix_crdb import is_crdb @@ -187,7 +187,7 @@ def test_diag_pickle(conn): (pq.__impl__ in ("c", "binary") and sys.version_info[:2] == (3, 12)), reason="Something with Exceptions, C, Python 3.12", ) -def test_diag_survives_cursor(conn): +def test_diag_survives_cursor(conn, gc_collect): cur = conn.cursor() with pytest.raises(e.Error) as exc: cur.execute("select * from nosuchtable") diff --git a/tests/test_server_cursor.py b/tests/test_server_cursor.py index 385dcd774..6b2ce2f14 100644 --- a/tests/test_server_cursor.py +++ b/tests/test_server_cursor.py @@ -251,11 +251,12 @@ def test_close_no_clobber(conn): cur.fetchall() -def test_warn_close(conn, recwarn): +def test_warn_close(conn, recwarn, gc_collect): recwarn.clear() cur = conn.cursor("foo") cur.execute("select generate_series(1, 10) as bar") del cur + gc_collect() assert ".close()" in str(recwarn.pop(ResourceWarning).message) diff --git a/tests/test_server_cursor_async.py b/tests/test_server_cursor_async.py index 99ff9cc76..a2e3cc90c 100644 --- a/tests/test_server_cursor_async.py +++ b/tests/test_server_cursor_async.py @@ -4,9 +4,7 @@ import psycopg from psycopg import rows, errors as e from psycopg.pq import Format -pytestmark = [ - pytest.mark.crdb_skip("server-side cursor"), -] +pytestmark = pytest.mark.crdb_skip("server-side cursor") async def test_init_row_factory(aconn): @@ -261,11 +259,12 @@ async def test_close_no_clobber(aconn): await cur.fetchall() -async def test_warn_close(aconn, recwarn): +async def test_warn_close(aconn, recwarn, gc_collect): recwarn.clear() cur = aconn.cursor("foo") await cur.execute("select generate_series(1, 10) as bar") del cur + gc_collect() assert ".close()" in str(recwarn.pop(ResourceWarning).message) diff --git a/tests/types/test_array.py b/tests/types/test_array.py index 8100bc7a8..8a40ab3b9 100644 --- a/tests/types/test_array.py +++ b/tests/types/test_array.py @@ -13,8 +13,6 @@ from psycopg._compat import prod from psycopg.postgres import types as builtins from psycopg.types.array import register_array -from ..utils import gc_collect - tests_str = [ ([[[[[["a"]]]]]], "{{{{{{a}}}}}}"), @@ -342,7 +340,7 @@ def test_all_chars_with_bounds(conn, fmt_out): @pytest.mark.slow -def test_register_array_leak(conn): +def test_register_array_leak(conn, gc_collect): info = TypeInfo.fetch(conn, "date") ntypes = [] for i in range(2): diff --git a/tests/utils.py b/tests/utils.py index 871f65d0e..064fab1c6 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,6 +1,4 @@ -import gc import re -import sys import operator from typing import Callable, Optional, Tuple @@ -135,45 +133,5 @@ class VersionCheck: return (ver_maj, ver_min, ver_fix) -def gc_collect(): - """ - gc.collect(), but more insisting. - """ - for i in range(3): - gc.collect() - - -NO_COUNT_TYPES: Tuple[type, ...] = () - -if sys.version_info[:2] == (3, 10): - # On my laptop there are occasional creations of a single one of these objects - # with empty content, which might be some Decimal caching. - # Keeping the guard as strict as possible, to be extended if other types - # or versions are necessary. - try: - from _contextvars import Context # type: ignore - except ImportError: - pass - else: - NO_COUNT_TYPES += (Context,) - - -def gc_count() -> int: - """ - len(gc.get_objects()), with subtleties. - """ - if not NO_COUNT_TYPES: - return len(gc.get_objects()) - - # Note: not using a list comprehension because it pollutes the objects list. - rv = 0 - for obj in gc.get_objects(): - if isinstance(obj, NO_COUNT_TYPES): - continue - rv += 1 - - return rv - - async def alist(it): return [i async for i in it] -- 2.47.2