From: Daniele Varrazzo Date: Tue, 13 Dec 2022 03:05:39 +0000 (+0000) Subject: test: ignore error looking for leaks X-Git-Tag: 3.1.5~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ff70a4d86f14e2658ee643c5696c525eabc3fc7a;p=thirdparty%2Fpsycopg.git test: ignore error looking for leaks The leaks counter frequently reports a leak between run 1 and 2 of the tests. It turns out to be a Decimal Context whose items() method reports no content. So the problem is related to Python, not psycopg, and it is not an unbound leak. Can't reproduce it in CI, but it happens on my laptop, since switching to Ubuntu 22.04 with Python 3.10. --- diff --git a/tests/crdb/test_copy.py b/tests/crdb/test_copy.py index 21eb78b60..b7d26aa51 100644 --- a/tests/crdb/test_copy.py +++ b/tests/crdb/test_copy.py @@ -1,4 +1,3 @@ -import gc import pytest import string from random import randrange, choice @@ -8,7 +7,7 @@ from psycopg.pq import Format from psycopg.adapt import PyFormat from psycopg.types.numeric import Int4 -from ..utils import eur, gc_collect +from ..utils import eur, gc_collect, gc_count from ..test_copy import sample_text, sample_binary # noqa from ..test_copy import ensure_table, sample_records from ..test_copy import sample_tabledef as sample_tabledef_pg @@ -225,7 +224,7 @@ def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): for i in range(3): work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/crdb/test_copy_async.py b/tests/crdb/test_copy_async.py index 7642b5635..d5fbf50d0 100644 --- a/tests/crdb/test_copy_async.py +++ b/tests/crdb/test_copy_async.py @@ -1,4 +1,3 @@ -import gc import pytest import string from random import randrange, choice @@ -8,7 +7,7 @@ from psycopg import sql, errors as e from psycopg.adapt import PyFormat from psycopg.types.numeric import Int4 -from ..utils import eur, gc_collect +from ..utils import eur, gc_collect, gc_count from ..test_copy import sample_text, sample_binary # noqa from ..test_copy import sample_records from ..test_copy_async import ensure_table @@ -231,6 +230,6 @@ async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): for i in range(3): await work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_client_cursor.py b/tests/test_client_cursor.py index 21fea8c4f..b35560474 100644 --- a/tests/test_client_cursor.py +++ b/tests/test_client_cursor.py @@ -1,4 +1,3 @@ -import gc import pickle import weakref import datetime as dt @@ -11,7 +10,7 @@ from psycopg import sql, rows from psycopg.adapt import PyFormat from psycopg.postgres import types as builtins -from .utils import gc_collect +from .utils import gc_collect, gc_count from .test_cursor import my_row_factory from .fix_crdb import is_crdb, crdb_encoding, crdb_time_precision @@ -806,7 +805,7 @@ def test_leak(conn_cls, dsn, faker, fetch, row_factory): for i in range(3): work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_client_cursor_async.py b/tests/test_client_cursor_async.py index 63e9c3cc9..0cf8ec649 100644 --- a/tests/test_client_cursor_async.py +++ b/tests/test_client_cursor_async.py @@ -1,4 +1,3 @@ -import gc import pytest import weakref import datetime as dt @@ -8,7 +7,7 @@ import psycopg from psycopg import sql, rows from psycopg.adapt import PyFormat -from .utils import alist, gc_collect +from .utils import alist, gc_collect, gc_count from .test_cursor import my_row_factory from .test_cursor import execmany, _execmany # noqa: F401 from .fix_crdb import crdb_encoding @@ -677,7 +676,7 @@ async def test_leak(aconn_cls, dsn, faker, fetch, row_factory): for i in range(3): await work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_copy.py b/tests/test_copy.py index 7844c3c4c..74e190fce 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -1,4 +1,3 @@ -import gc import string import struct import hashlib @@ -19,7 +18,7 @@ from psycopg.types import TypeInfo from psycopg.types.hstore import register_hstore from psycopg.types.numeric import Int4 -from .utils import eur, gc_collect +from .utils import eur, gc_collect, gc_count pytestmark = pytest.mark.crdb_skip("copy") @@ -727,7 +726,7 @@ def test_copy_to_leaks(conn_cls, dsn, faker, fmt, set_types, method): for i in range(3): work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" @@ -770,7 +769,7 @@ def test_copy_from_leaks(conn_cls, dsn, faker, fmt, set_types): for i in range(3): work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_copy_async.py b/tests/test_copy_async.py index 24aa9cabf..9b926a213 100644 --- a/tests/test_copy_async.py +++ b/tests/test_copy_async.py @@ -1,4 +1,3 @@ -import gc import string import hashlib from io import BytesIO, StringIO @@ -19,7 +18,7 @@ from psycopg.adapt import PyFormat from psycopg.types.hstore import register_hstore from psycopg.types.numeric import Int4 -from .utils import alist, eur, gc_collect +from .utils import alist, eur, gc_collect, gc_count from .test_copy import sample_text, sample_binary, sample_binary_rows # noqa from .test_copy import sample_values, sample_records, sample_tabledef from .test_copy import py_to_raw, special_chars @@ -734,7 +733,7 @@ async def test_copy_to_leaks(aconn_cls, dsn, faker, fmt, set_types, method): for i in range(3): await work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" @@ -777,7 +776,7 @@ async def test_copy_from_leaks(aconn_cls, dsn, faker, fmt, set_types): for i in range(3): await work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 8b044944a..a667f4fb3 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -1,4 +1,3 @@ -import gc import pickle import weakref import datetime as dt @@ -13,7 +12,7 @@ from psycopg.adapt import PyFormat from psycopg.postgres import types as builtins from psycopg.rows import RowMaker -from .utils import gc_collect +from .utils import gc_collect, gc_count from .fix_crdb import is_crdb, crdb_encoding, crdb_time_precision @@ -925,7 +924,7 @@ def test_leak(conn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): for i in range(3): work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/test_cursor_async.py b/tests/test_cursor_async.py index 84b572260..ac3fdeb2c 100644 --- a/tests/test_cursor_async.py +++ b/tests/test_cursor_async.py @@ -1,4 +1,3 @@ -import gc import pytest import weakref import datetime as dt @@ -8,7 +7,7 @@ import psycopg from psycopg import pq, sql, rows from psycopg.adapt import PyFormat -from .utils import gc_collect +from .utils import gc_collect, gc_count from .test_cursor import my_row_factory from .test_cursor import execmany, _execmany # noqa: F401 from .fix_crdb import crdb_encoding @@ -798,6 +797,6 @@ async def test_leak(aconn_cls, dsn, faker, fmt, fmt_out, fetch, row_factory): for i in range(3): await work() gc_collect() - n.append(len(gc.get_objects())) + n.append(gc_count()) assert n[0] == n[1] == n[2], f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" diff --git a/tests/utils.py b/tests/utils.py index 47c796629..871f65d0e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,6 @@ import gc import re +import sys import operator from typing import Callable, Optional, Tuple @@ -142,5 +143,37 @@ def gc_collect(): gc.collect() +NO_COUNT_TYPES: Tuple[type, ...] = () + +if sys.version_info[:2] == (3, 10): + # On my laptop there are occasional creations of a single one of these objects + # with empty content, which might be some Decimal caching. + # Keeping the guard as strict as possible, to be extended if other types + # or versions are necessary. + try: + from _contextvars import Context # type: ignore + except ImportError: + pass + else: + NO_COUNT_TYPES += (Context,) + + +def gc_count() -> int: + """ + len(gc.get_objects()), with subtleties. + """ + if not NO_COUNT_TYPES: + return len(gc.get_objects()) + + # Note: not using a list comprehension because it pollutes the objects list. + rv = 0 + for obj in gc.get_objects(): + if isinstance(obj, NO_COUNT_TYPES): + continue + rv += 1 + + return rv + + async def alist(it): return [i async for i in it]