From: Daniele Varrazzo Date: Sat, 16 Jan 2021 02:51:19 +0000 (+0100) Subject: Added tests to detect leaks on copy from X-Git-Tag: 3.0.dev0~147 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a52743137aac689853978846f34eac9adcb33143;p=thirdparty%2Fpsycopg.git Added tests to detect leaks on copy from --- diff --git a/tests/fix_faker.py b/tests/fix_faker.py index d6928ae86..fbd806bec 100644 --- a/tests/fix_faker.py +++ b/tests/fix_faker.py @@ -29,7 +29,7 @@ class Faker: def __init__(self, connection): self.conn = connection - self.format = Format.AUTO + self._format = Format.BINARY self.records = [] self._schema = None @@ -37,6 +37,16 @@ class Faker: self._makers = {} self.table_name = sql.Identifier("fake_table") + @property + def format(self): + return self._format + + @format.setter + def format(self, format): + if format != Format.BINARY: + pytest.xfail("faker to extend to all text dumpers") + self._format = format + @property def schema(self): if not self._schema: diff --git a/tests/test_adapt.py b/tests/test_adapt.py index 222538cda..5c6023675 100644 --- a/tests/test_adapt.py +++ b/tests/test_adapt.py @@ -294,9 +294,6 @@ def test_optimised_adapters(): @pytest.mark.slow @pytest.mark.parametrize("fmt", [Format.AUTO, Format.TEXT, Format.BINARY]) def test_random(conn, faker, fmt): - if fmt != Format.BINARY: - pytest.xfail("faker to extend to all text dumpers") - faker.format = fmt faker.choose_schema(ncols=20) faker.make_records(50) diff --git a/tests/test_copy.py b/tests/test_copy.py index bcf2ec2fc..0ff86ea83 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -500,9 +500,6 @@ def test_worker_life(conn, format, buffer): @pytest.mark.parametrize("fmt", [Format.TEXT, Format.BINARY]) @pytest.mark.parametrize("method", ["read", "iter", "row", "rows"]) def test_copy_to_leaks(dsn, faker, fmt, method): - if fmt != Format.BINARY: - pytest.xfail("faker to extend to all text dumpers") - faker.format = PgFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -557,6 +554,47 @@ def test_copy_to_leaks(dsn, faker, fmt, method): ), f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" +@pytest.mark.slow +@pytest.mark.parametrize("fmt", [Format.TEXT, Format.BINARY]) +def test_copy_from_leaks(dsn, faker, fmt): + faker.format = PgFormat.from_pq(fmt) + faker.choose_schema(ncols=20) + faker.make_records(20) + + n = [] + for i in range(3): + with psycopg3.connect(dsn) as conn: + with conn.cursor(binary=fmt) as cur: + cur.execute(faker.drop_stmt) + cur.execute(faker.create_stmt) + + stmt = sql.SQL("copy {} ({}) from stdin (format {})").format( + faker.table_name, + sql.SQL(", ").join(faker.fields_names), + sql.SQL(fmt.name), + ) + with cur.copy(stmt) as copy: + for row in faker.records: + copy.write_row(row) + + cur.execute(faker.select_stmt) + recs = cur.fetchall() + + for got, want in zip(recs, faker.records): + faker.assert_record(got, want) + + del recs + + del cur, conn + gc.collect() + gc.collect() + n.append(len(gc.get_objects())) + + assert ( + n[0] == n[1] == n[2] + ), f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" + + def py_to_raw(item, fmt): """Convert from Python type to the expected result from the db""" if fmt == Format.TEXT: diff --git a/tests/test_copy_async.py b/tests/test_copy_async.py index 69d61772b..236b86d6d 100644 --- a/tests/test_copy_async.py +++ b/tests/test_copy_async.py @@ -479,9 +479,6 @@ async def test_worker_life(aconn, format, buffer): @pytest.mark.parametrize("fmt", [Format.TEXT, Format.BINARY]) @pytest.mark.parametrize("method", ["read", "iter", "row", "rows"]) async def test_copy_to_leaks(dsn, faker, fmt, method): - if fmt != Format.BINARY: - pytest.xfail("faker to extend to all text dumpers") - faker.format = PgFormat.from_pq(fmt) faker.choose_schema(ncols=20) faker.make_records(20) @@ -538,6 +535,47 @@ async def test_copy_to_leaks(dsn, faker, fmt, method): ), f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" +@pytest.mark.slow +@pytest.mark.parametrize("fmt", [Format.TEXT, Format.BINARY]) +async def test_copy_from_leaks(dsn, faker, fmt): + faker.format = PgFormat.from_pq(fmt) + faker.choose_schema(ncols=20) + faker.make_records(20) + + n = [] + for i in range(3): + async with await psycopg3.AsyncConnection.connect(dsn) as conn: + async with await conn.cursor(binary=fmt) as cur: + await cur.execute(faker.drop_stmt) + await cur.execute(faker.create_stmt) + + stmt = sql.SQL("copy {} ({}) from stdin (format {})").format( + faker.table_name, + sql.SQL(", ").join(faker.fields_names), + sql.SQL(fmt.name), + ) + async with cur.copy(stmt) as copy: + for row in faker.records: + await copy.write_row(row) + + await cur.execute(faker.select_stmt) + recs = await cur.fetchall() + + for got, want in zip(recs, faker.records): + faker.assert_record(got, want) + + del recs + + del cur, conn + gc.collect() + gc.collect() + n.append(len(gc.get_objects())) + + assert ( + n[0] == n[1] == n[2] + ), f"objects leaked: {n[1] - n[0]}, {n[2] - n[1]}" + + async def ensure_table(cur, tabledef, name="copy_in"): await cur.execute(f"drop table if exists {name}") await cur.execute(f"create table {name} ({tabledef})") diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 11629afc4..5b9a3e57b 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -407,9 +407,6 @@ def test_str(conn): @pytest.mark.parametrize("fmt", [Format.AUTO, Format.TEXT, Format.BINARY]) @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) def test_leak(dsn, faker, fmt, fetch): - if fmt != Format.BINARY: - pytest.xfail("faker to extend to all text dumpers") - faker.format = fmt faker.choose_schema(ncols=5) faker.make_records(10) diff --git a/tests/test_cursor_async.py b/tests/test_cursor_async.py index c9b7f1fee..e73b9ea08 100644 --- a/tests/test_cursor_async.py +++ b/tests/test_cursor_async.py @@ -320,9 +320,6 @@ async def test_str(aconn): @pytest.mark.parametrize("fmt", [Format.AUTO, Format.TEXT, Format.BINARY]) @pytest.mark.parametrize("fetch", ["one", "many", "all", "iter"]) async def test_leak(dsn, faker, fmt, fetch): - if fmt != Format.BINARY: - pytest.xfail("faker to extend to all text dumpers") - faker.format = fmt faker.choose_schema(ncols=5) faker.make_records(10)